simd-vector-engine 0.1.0

Runtime-dispatched SIMD kernels for scanning i32/f32 slices: count, sum, find and stream compaction on AVX-512, AVX2, NEON with a scalar fallback
Documentation
// NEON, 128-битные Q-регистры, 4 лейна по 32 бита. Только aarch64.
// vld1q_s32 не требует выравнивания, movemask нет — вместо него vaddvq_*/vmaxvq_*.

use core::arch::aarch64::*;

/// # Safety
///
/// The `neon` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "neon")]
pub unsafe fn count_gt_i32(data: &[i32], threshold: i32) -> usize {
    let n = data.len();
    let p = data.as_ptr();

    let thr = vdupq_n_s32(threshold); // dup v.4s, w
    let mut a0 = vdupq_n_s32(0);
    let mut a1 = vdupq_n_s32(0);
    let mut a2 = vdupq_n_s32(0);
    let mut a3 = vdupq_n_s32(0);

    let mut i = 0usize;
    let block = n & !15usize;
    while i < block {
        // ld1 {v.4s}
        let v0 = vld1q_s32(p.add(i));
        let v1 = vld1q_s32(p.add(i + 4));
        let v2 = vld1q_s32(p.add(i + 8));
        let v3 = vld1q_s32(p.add(i + 12));

        // cmgt v.4s: лейн = -1 при истине
        let m0 = vreinterpretq_s32_u32(vcgtq_s32(v0, thr));
        let m1 = vreinterpretq_s32_u32(vcgtq_s32(v1, thr));
        let m2 = vreinterpretq_s32_u32(vcgtq_s32(v2, thr));
        let m3 = vreinterpretq_s32_u32(vcgtq_s32(v3, thr));

        // sub v.4s: acc - (-1) == acc + 1
        a0 = vsubq_s32(a0, m0);
        a1 = vsubq_s32(a1, m1);
        a2 = vsubq_s32(a2, m2);
        a3 = vsubq_s32(a3, m3);

        i += 16;
    }

    let acc = vaddq_s32(vaddq_s32(a0, a1), vaddq_s32(a2, a3));
    // addv s0, v.4s: горизонтальная сумма одной инструкцией
    let mut total = vaddvq_s32(acc) as usize;

    while i < n {
        total += (*p.add(i) > threshold) as usize;
        i += 1;
    }
    total
}

/// # Safety
///
/// The `neon` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "neon")]
pub unsafe fn count_gt_f32(data: &[f32], threshold: f32) -> usize {
    let n = data.len();
    let p = data.as_ptr();

    let thr = vdupq_n_f32(threshold);
    let mut a0 = vdupq_n_s32(0);
    let mut a1 = vdupq_n_s32(0);

    let mut i = 0usize;
    let block = n & !7usize;
    while i < block {
        let v0 = vld1q_f32(p.add(i));
        let v1 = vld1q_f32(p.add(i + 4));

        // fcmgt v.4s: NaN даёт false, как `>` в Rust
        let m0 = vreinterpretq_s32_u32(vcgtq_f32(v0, thr));
        let m1 = vreinterpretq_s32_u32(vcgtq_f32(v1, thr));

        a0 = vsubq_s32(a0, m0);
        a1 = vsubq_s32(a1, m1);
        i += 8;
    }

    let mut total = vaddvq_s32(vaddq_s32(a0, a1)) as usize;
    while i < n {
        total += (*p.add(i) > threshold) as usize;
        i += 1;
    }
    total
}

/// # Safety
///
/// The `neon` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "neon")]
pub unsafe fn sum_i32(data: &[i32]) -> i64 {
    let n = data.len();
    let p = data.as_ptr();

    let mut a0 = vdupq_n_s64(0);
    let mut a1 = vdupq_n_s64(0);

    let mut i = 0usize;
    let block = n & !7usize;
    while i < block {
        let v0 = vld1q_s32(p.add(i));
        let v1 = vld1q_s32(p.add(i + 4));

        // saddlp: попарное сложение с расширением 4×i32 -> 2×i64,
        // одна инструкция вместо связки vpmovsxdq+vpaddq из x86
        a0 = vaddq_s64(a0, vpaddlq_s32(v0));
        a1 = vaddq_s64(a1, vpaddlq_s32(v1));
        i += 8;
    }

    let mut total = vaddvq_s64(vaddq_s64(a0, a1));
    while i < n {
        total += *p.add(i) as i64;
        i += 1;
    }
    total
}

/// # Safety
///
/// The `neon` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "neon")]
pub unsafe fn find_i32(data: &[i32], needle: i32) -> Option<usize> {
    let n = data.len();
    let p = data.as_ptr();
    let key = vdupq_n_s32(needle);

    let mut i = 0usize;
    let block = n & !15usize;
    while i < block {
        let v0 = vld1q_s32(p.add(i));
        let v1 = vld1q_s32(p.add(i + 4));
        let v2 = vld1q_s32(p.add(i + 8));
        let v3 = vld1q_s32(p.add(i + 12));

        // cmeq v.4s
        let e0 = vceqq_s32(v0, key);
        let e1 = vceqq_s32(v1, key);
        let e2 = vceqq_s32(v2, key);
        let e3 = vceqq_s32(v3, key);

        // orr × 3, затем umaxv: максимум 0 означает отсутствие совпадений
        let any = vorrq_u32(vorrq_u32(e0, e1), vorrq_u32(e2, e3));
        if vmaxvq_u32(any) != 0 {
            for (k, e) in [e0, e1, e2, e3].into_iter().enumerate() {
                let mut lanes = [0u32; 4];
                vst1q_u32(lanes.as_mut_ptr(), e);
                for (l, &val) in lanes.iter().enumerate() {
                    if val != 0 {
                        return Some(i + k * 4 + l);
                    }
                }
            }
        }
        i += 16;
    }

    while i < n {
        if *p.add(i) == needle {
            return Some(i);
        }
        i += 1;
    }
    None
}

// Аналога vpcompressd в базовом NEON нет (только SVE2 compact),
// поэтому вектор отсеивает пустые блоки, а запись скалярная.
/// # Safety
///
/// The `neon` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "neon")]
pub unsafe fn filter_into_i32(data: &[i32], threshold: i32, out: &mut [i32]) -> usize {
    debug_assert!(data.is_empty() || !out.is_empty());
    let n = data.len();
    let p = data.as_ptr();
    let o = out.as_mut_ptr();
    let thr = vdupq_n_s32(threshold);

    let mut w = 0usize;
    let mut i = 0usize;
    let block = n & !3usize;
    while i < block {
        let v = vld1q_s32(p.add(i));
        let m = vcgtq_s32(v, thr);
        if vmaxvq_u32(m) != 0 {
            let mut lanes = [0i32; 4];
            let mut flags = [0u32; 4];
            vst1q_s32(lanes.as_mut_ptr(), v);
            vst1q_u32(flags.as_mut_ptr(), m);
            for l in 0..4 {
                if flags[l] != 0 {
                    *o.add(w) = lanes[l];
                    w += 1;
                }
            }
        }
        i += 4;
    }

    while i < n {
        let x = *p.add(i);
        if x > threshold {
            *o.add(w) = x;
            w += 1;
        }
        i += 1;
    }
    w
}