scanline-simd 0.1.0-alpha.0

Vectorised scan predicates for Scanline — AVX-512, AVX2, NEON, and a portable fallback behind one safe API.
Documentation
//! x86-64 intrinsic backends.
//!
//! Note on the source document's sample: it used
//! `_mm512_cmp_epi64_mask::<_MM_CMPINT_GT>(..)`, which does not compile —
//! `_MM_CMPINT_GT` is not a constant in Rust's `std::arch` (it is a type alias
//! there). The correct spellings are the dedicated `_mm512_cmpgt_epi64_mask`
//! used below, or `_mm512_cmp_epi64_mask::<6>` with the raw predicate constant.
//!
//! Note on expected speedup: a 64-bit scan moves 8 bytes per element, so at the
//! target rate this kernel is memory-bandwidth-bound roughly 30x over before it
//! is ALU-bound. On Zen 4, which executes 512-bit operations on a 256-bit
//! datapath, the AVX-512 path should be expected to land close to AVX2 rather
//! than 2x ahead. Vector width here buys *fewer cores needed to saturate the
//! bus*, not a higher ceiling.

#![cfg(target_arch = "x86_64")]

use std::arch::x86_64::{
    __m256i, _mm256_cmpgt_epi64, _mm256_loadu_si256, _mm256_movemask_pd, _mm256_set1_epi64x,
    _mm512_cmpgt_epi64_mask, _mm512_loadu_si512, _mm512_set1_epi64,
};

/// Number of `i64` lanes in a ZMM register.
const AVX512_LANES: usize = 8;
/// Number of `i64` lanes in a YMM register.
const AVX2_LANES: usize = 4;

/// Counts elements of `data` strictly greater than `target`, using AVX-512F.
///
/// # Safety
///
/// The caller must ensure the host CPU supports `avx512f`. Callers in this crate
/// guarantee it with `is_x86_feature_detected!("avx512f")`.
#[target_feature(enable = "avx512f")]
pub unsafe fn scan_gt_i64_avx512(data: &[i64], target: i64) -> usize {
    let mut matches = 0_usize;

    // Register-only intrinsics are safe to call inside a `#[target_feature]` fn —
    // the feature gate is what made them unsafe. Only the pointer loads below
    // still need an explicit unsafe block.
    let target_vec = _mm512_set1_epi64(target);

    let chunks = data.chunks_exact(AVX512_LANES);
    // `chunks_exact` drops any trailing partial chunk, so the remainder is
    // captured here and counted scalar-wise below. Omitting this is the bug in
    // the source document's sample.
    let remainder = chunks.remainder();

    for chunk in chunks {
        // SAFETY: `chunk` is exactly AVX512_LANES i64 = 64 bytes, so the
        // unaligned 512-bit load reads entirely within the slice. `avx512f` is
        // guaranteed by this function's contract.
        let data_vec = unsafe { _mm512_loadu_si512(chunk.as_ptr().cast()) };
        let mask: u8 = _mm512_cmpgt_epi64_mask(data_vec, target_vec);
        matches += mask.count_ones() as usize;
    }

    matches + super::portable::scan_gt_i64(remainder, target)
}

/// Counts elements of `data` strictly greater than `target`, using AVX2.
///
/// # Safety
///
/// The caller must ensure the host CPU supports `avx2`. Callers in this crate
/// guarantee it with `is_x86_feature_detected!("avx2")`.
#[target_feature(enable = "avx2")]
pub unsafe fn scan_gt_i64_avx2(data: &[i64], target: i64) -> usize {
    let mut matches = 0_usize;

    // Safe inside a `#[target_feature]` fn — see the note in the AVX-512 path.
    let target_vec = _mm256_set1_epi64x(target);

    let chunks = data.chunks_exact(AVX2_LANES);
    let remainder = chunks.remainder();

    for chunk in chunks {
        // SAFETY: `chunk` is exactly AVX2_LANES i64 = 32 bytes, so the unaligned
        // 256-bit load reads entirely within the slice. `avx2` is guaranteed by
        // this function's contract.
        let data_vec: __m256i = unsafe { _mm256_loadu_si256(chunk.as_ptr().cast()) };
        let cmp = _mm256_cmpgt_epi64(data_vec, target_vec);
        // AVX2 has no integer movemask for 64-bit lanes. Reinterpreting the
        // all-ones/all-zeros comparison result as f64 and taking the sign-bit
        // mask yields one bit per lane, which is the standard idiom here. The bit
        // pattern is only masked, never interpreted as a number, so non-canonical
        // float patterns are irrelevant.
        // SAFETY: `__m256i` and `__m256d` are both 32-byte vector types with the
        // same size and alignment; this is a reinterpretation, not a conversion.
        let as_pd = unsafe { std::mem::transmute::<__m256i, std::arch::x86_64::__m256d>(cmp) };
        let bits = _mm256_movemask_pd(as_pd);
        matches += bits.unsigned_abs().count_ones() as usize;
    }

    matches + super::portable::scan_gt_i64(remainder, target)
}