Skip to main content

rustbinary/
simd.rs

1//! Runtime-dispatched SIMD primitives used by codec hot paths.
2
3/// SIMD implementation selected for byte-oriented codec operations.
4#[derive(Clone, Copy, Debug, Eq, PartialEq)]
5#[non_exhaustive]
6pub enum SimdBackend {
7    /// Portable scalar implementation.
8    Scalar,
9    /// 128-bit x86 SSE2 implementation.
10    Sse2,
11    /// 256-bit x86 AVX2 implementation.
12    Avx2,
13    /// 128-bit AArch64 NEON implementation.
14    Neon,
15}
16
17/// Hardware features relevant to serialization workloads.
18///
19/// A capability can be present without being selected as [`SimdBackend`]. For
20/// example, AVX-512, SVE, and SME do not currently have codec kernels because
21/// wider state can reduce clock speed and SME is intended for matrix tiles.
22#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
23pub struct HardwareCapabilities {
24    /// x86 SSE2 support.
25    pub sse2: bool,
26    /// x86 AVX support, including operating-system extended-state support.
27    pub avx: bool,
28    /// x86 AVX2 support.
29    pub avx2: bool,
30    /// x86 AVX-512 foundation support.
31    pub avx512f: bool,
32    /// x86 AVX-512 byte/word support.
33    pub avx512bw: bool,
34    /// AArch64 NEON support.
35    pub neon: bool,
36    /// AArch64 scalable vector extension support.
37    pub sve: bool,
38    /// AArch64 scalable matrix extension support.
39    pub sme: bool,
40}
41
42/// Detects hardware SIMD capabilities at runtime.
43pub fn hardware_capabilities() -> HardwareCapabilities {
44    let mut capabilities = HardwareCapabilities::default();
45    #[cfg(target_arch = "x86_64")]
46    {
47        capabilities.sse2 = std::arch::is_x86_feature_detected!("sse2");
48        capabilities.avx = std::arch::is_x86_feature_detected!("avx");
49        capabilities.avx2 = std::arch::is_x86_feature_detected!("avx2");
50        capabilities.avx512f = std::arch::is_x86_feature_detected!("avx512f");
51        capabilities.avx512bw = std::arch::is_x86_feature_detected!("avx512bw");
52    }
53    #[cfg(target_arch = "aarch64")]
54    {
55        capabilities.neon = std::arch::is_aarch64_feature_detected!("neon");
56        capabilities.sve = std::arch::is_aarch64_feature_detected!("sve");
57        capabilities.sme = detect_sme();
58    }
59    capabilities
60}
61
62#[cfg(all(
63    target_arch = "aarch64",
64    any(target_os = "linux", target_os = "android")
65))]
66fn detect_sme() -> bool {
67    use std::ffi::c_ulong;
68
69    const AT_HWCAP2: c_ulong = 26;
70    const HWCAP2_SME: c_ulong = 1 << 23;
71
72    unsafe extern "C" {
73        fn getauxval(kind: c_ulong) -> c_ulong;
74    }
75
76    // SAFETY: `getauxval` has no pointer arguments and is provided by libc on
77    // Linux and Android. A missing capability is reported as a zero bit.
78    unsafe { getauxval(AT_HWCAP2) & HWCAP2_SME != 0 }
79}
80
81#[cfg(all(
82    target_arch = "aarch64",
83    not(any(target_os = "linux", target_os = "android"))
84))]
85const fn detect_sme() -> bool {
86    false
87}
88
89/// Returns the fastest implemented backend supported by the current process.
90///
91/// Runtime feature probing is performed once and cached: every codec hot-path
92/// dispatch (`is_ascii`, `plain_varint_prefix`) reuses the result instead of
93/// re-running CPU feature detection on each call.
94pub fn simd_backend() -> SimdBackend {
95    // `OnceLock` (1.70) instead of `LazyLock` (1.80) keeps the MSRV at 1.78.
96    static BACKEND: std::sync::OnceLock<SimdBackend> = std::sync::OnceLock::new();
97    *BACKEND.get_or_init(|| {
98        let capabilities = hardware_capabilities();
99        if capabilities.avx2 {
100            SimdBackend::Avx2
101        } else if capabilities.sse2 {
102            SimdBackend::Sse2
103        } else if capabilities.neon {
104            SimdBackend::Neon
105        } else {
106            SimdBackend::Scalar
107        }
108    })
109}
110
111#[cfg(feature = "adaptive")]
112pub(crate) fn is_ascii(bytes: &[u8]) -> bool {
113    first_non_ascii(bytes) == bytes.len()
114}
115
116#[cfg(feature = "adaptive")]
117pub(crate) fn plain_varint_prefix(bytes: &[u8]) -> usize {
118    match simd_backend() {
119        #[cfg(target_arch = "x86_64")]
120        SimdBackend::Avx2 => {
121            // SAFETY: runtime detection above proves AVX2 availability. The
122            // kernel uses unaligned loads only within complete 32-byte chunks.
123            unsafe { x86::plain_prefix_avx2(bytes) }
124        }
125        #[cfg(target_arch = "x86_64")]
126        SimdBackend::Sse2 => {
127            // SAFETY: SSE2 is checked at runtime and each load is in bounds.
128            unsafe { x86::plain_prefix_sse2(bytes) }
129        }
130        #[cfg(target_arch = "aarch64")]
131        SimdBackend::Neon => {
132            // SAFETY: NEON is checked at runtime and each load is in bounds.
133            unsafe { arm::plain_prefix_neon(bytes) }
134        }
135        _ => scalar_plain_prefix(bytes),
136    }
137}
138
139#[cfg(feature = "adaptive")]
140fn first_non_ascii(bytes: &[u8]) -> usize {
141    match simd_backend() {
142        #[cfg(target_arch = "x86_64")]
143        SimdBackend::Avx2 => {
144            // SAFETY: runtime feature detection and bounds are enforced here.
145            unsafe { x86::ascii_prefix_avx2(bytes) }
146        }
147        #[cfg(target_arch = "x86_64")]
148        SimdBackend::Sse2 => {
149            // SAFETY: runtime feature detection and bounds are enforced here.
150            unsafe { x86::ascii_prefix_sse2(bytes) }
151        }
152        #[cfg(target_arch = "aarch64")]
153        SimdBackend::Neon => {
154            // SAFETY: runtime feature detection and bounds are enforced here.
155            unsafe { arm::ascii_prefix_neon(bytes) }
156        }
157        _ => scalar_ascii_prefix(bytes),
158    }
159}
160
161#[cfg(feature = "adaptive")]
162fn scalar_ascii_prefix(bytes: &[u8]) -> usize {
163    bytes
164        .iter()
165        .position(|byte| !byte.is_ascii())
166        .unwrap_or(bytes.len())
167}
168
169#[cfg(feature = "adaptive")]
170fn scalar_plain_prefix(bytes: &[u8]) -> usize {
171    bytes
172        .iter()
173        .position(|byte| *byte > 250)
174        .unwrap_or(bytes.len())
175}
176
177#[cfg(all(feature = "adaptive", target_arch = "x86_64"))]
178mod x86 {
179    // The compute intrinsics (set1 / xor / cmpgt / movemask) are `unsafe fn` on
180    // Rust 1.78 but safe to call in newer releases. The blocks below keep the
181    // code compiling on both, and this allow silences the redundant-unsafe
182    // warning that newer Rust would otherwise emit.
183    #![allow(unused_unsafe)]
184    use std::arch::x86_64::*;
185
186    #[target_feature(enable = "avx2")]
187    pub(super) unsafe fn ascii_prefix_avx2(bytes: &[u8]) -> usize {
188        let mut offset = 0;
189        while offset + 32 <= bytes.len() {
190            // SAFETY: the loop proves that a complete unaligned vector is readable.
191            let vector = unsafe { _mm256_loadu_si256(bytes.as_ptr().add(offset).cast()) };
192            let mask = unsafe { _mm256_movemask_epi8(vector) } as u32;
193            if mask != 0 {
194                return offset + mask.trailing_zeros() as usize;
195            }
196            offset += 32;
197        }
198        offset + super::scalar_ascii_prefix(&bytes[offset..])
199    }
200
201    #[target_feature(enable = "sse2")]
202    pub(super) unsafe fn ascii_prefix_sse2(bytes: &[u8]) -> usize {
203        let mut offset = 0;
204        while offset + 16 <= bytes.len() {
205            // SAFETY: the loop proves that a complete unaligned vector is readable.
206            let vector = unsafe { _mm_loadu_si128(bytes.as_ptr().add(offset).cast()) };
207            let mask = unsafe { _mm_movemask_epi8(vector) } as u32;
208            if mask != 0 {
209                return offset + mask.trailing_zeros() as usize;
210            }
211            offset += 16;
212        }
213        offset + super::scalar_ascii_prefix(&bytes[offset..])
214    }
215
216    #[target_feature(enable = "avx2")]
217    pub(super) unsafe fn plain_prefix_avx2(bytes: &[u8]) -> usize {
218        let mut offset = 0;
219        let sign = unsafe { _mm256_set1_epi8(i8::MIN) };
220        let threshold = unsafe { _mm256_set1_epi8(122) };
221        while offset + 32 <= bytes.len() {
222            // SAFETY: the loop proves that a complete unaligned vector is readable.
223            let vector = unsafe { _mm256_loadu_si256(bytes.as_ptr().add(offset).cast()) };
224            let unsigned_order = unsafe { _mm256_xor_si256(vector, sign) };
225            let markers = unsafe { _mm256_cmpgt_epi8(unsigned_order, threshold) };
226            let mask = unsafe { _mm256_movemask_epi8(markers) } as u32;
227            if mask != 0 {
228                return offset + mask.trailing_zeros() as usize;
229            }
230            offset += 32;
231        }
232        offset + super::scalar_plain_prefix(&bytes[offset..])
233    }
234
235    #[target_feature(enable = "sse2")]
236    pub(super) unsafe fn plain_prefix_sse2(bytes: &[u8]) -> usize {
237        let mut offset = 0;
238        let sign = unsafe { _mm_set1_epi8(i8::MIN) };
239        let threshold = unsafe { _mm_set1_epi8(122) };
240        while offset + 16 <= bytes.len() {
241            // SAFETY: the loop proves that a complete unaligned vector is readable.
242            let vector = unsafe { _mm_loadu_si128(bytes.as_ptr().add(offset).cast()) };
243            let unsigned_order = unsafe { _mm_xor_si128(vector, sign) };
244            let markers = unsafe { _mm_cmpgt_epi8(unsigned_order, threshold) };
245            let mask = unsafe { _mm_movemask_epi8(markers) } as u32;
246            if mask != 0 {
247                return offset + mask.trailing_zeros() as usize;
248            }
249            offset += 16;
250        }
251        offset + super::scalar_plain_prefix(&bytes[offset..])
252    }
253}
254
255#[cfg(all(feature = "adaptive", target_arch = "aarch64"))]
256mod arm {
257    // See the `x86` module: NEON compute intrinsics are `unsafe fn` on Rust
258    // 1.78 but safe in newer releases.
259    #![allow(unused_unsafe)]
260    use std::arch::aarch64::*;
261
262    #[target_feature(enable = "neon")]
263    pub(super) unsafe fn ascii_prefix_neon(bytes: &[u8]) -> usize {
264        let mut offset = 0;
265        let threshold = unsafe { vdupq_n_u8(127) };
266        while offset + 16 <= bytes.len() {
267            // SAFETY: the loop proves that a complete unaligned vector is readable.
268            let vector = unsafe { vld1q_u8(bytes.as_ptr().add(offset)) };
269            let compared = unsafe { vcgtq_u8(vector, threshold) };
270            let mut lanes = [0u8; 16];
271            // SAFETY: `lanes` has exactly one vector of writable storage.
272            unsafe { vst1q_u8(lanes.as_mut_ptr(), compared) };
273            if let Some(index) = lanes.iter().position(|lane| *lane != 0) {
274                return offset + index;
275            }
276            offset += 16;
277        }
278        offset + super::scalar_ascii_prefix(&bytes[offset..])
279    }
280
281    #[target_feature(enable = "neon")]
282    pub(super) unsafe fn plain_prefix_neon(bytes: &[u8]) -> usize {
283        let mut offset = 0;
284        let threshold = unsafe { vdupq_n_u8(250) };
285        while offset + 16 <= bytes.len() {
286            // SAFETY: the loop proves that a complete unaligned vector is readable.
287            let vector = unsafe { vld1q_u8(bytes.as_ptr().add(offset)) };
288            let compared = unsafe { vcgtq_u8(vector, threshold) };
289            let mut lanes = [0u8; 16];
290            // SAFETY: `lanes` has exactly one vector of writable storage.
291            unsafe { vst1q_u8(lanes.as_mut_ptr(), compared) };
292            if let Some(index) = lanes.iter().position(|lane| *lane != 0) {
293                return offset + index;
294            }
295            offset += 16;
296        }
297        offset + super::scalar_plain_prefix(&bytes[offset..])
298    }
299}
300
301#[cfg(all(test, feature = "adaptive"))]
302mod tests {
303    use super::*;
304    use alloc::vec;
305    use alloc::vec::Vec;
306
307    #[test]
308    fn dispatched_scanners_match_scalar_at_vector_boundaries() {
309        for length in 0..96 {
310            let ascii = vec![b'a'; length];
311            assert_eq!(first_non_ascii(&ascii), scalar_ascii_prefix(&ascii));
312            assert_eq!(plain_varint_prefix(&ascii), scalar_plain_prefix(&ascii));
313
314            for position in 0..length {
315                let mut non_ascii = ascii.clone();
316                non_ascii[position] = 0x80;
317                assert_eq!(first_non_ascii(&non_ascii), position);
318
319                let mut marker = ascii.clone();
320                marker[position] = 251;
321                assert_eq!(plain_varint_prefix(&marker), position);
322            }
323        }
324    }
325
326    #[test]
327    fn plain_varint_scan_accepts_entire_single_byte_domain() {
328        let values = (0..=250).map(|value| value as u8).collect::<Vec<_>>();
329        assert_eq!(plain_varint_prefix(&values), values.len());
330        for marker in 251..=255 {
331            let mut input = values.clone();
332            input.push(marker);
333            assert_eq!(plain_varint_prefix(&input), values.len());
334        }
335    }
336}