1#[derive(Clone, Copy, Debug, Eq, PartialEq)]
5#[non_exhaustive]
6pub enum SimdBackend {
7 Scalar,
9 Sse2,
11 Avx2,
13 Neon,
15}
16
17#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
23pub struct HardwareCapabilities {
24 pub sse2: bool,
26 pub avx: bool,
28 pub avx2: bool,
30 pub avx512f: bool,
32 pub avx512bw: bool,
34 pub neon: bool,
36 pub sve: bool,
38 pub sme: bool,
40}
41
42pub fn hardware_capabilities() -> HardwareCapabilities {
44 let mut capabilities = HardwareCapabilities::default();
45 #[cfg(target_arch = "x86_64")]
46 {
47 capabilities.sse2 = std::arch::is_x86_feature_detected!("sse2");
48 capabilities.avx = std::arch::is_x86_feature_detected!("avx");
49 capabilities.avx2 = std::arch::is_x86_feature_detected!("avx2");
50 capabilities.avx512f = std::arch::is_x86_feature_detected!("avx512f");
51 capabilities.avx512bw = std::arch::is_x86_feature_detected!("avx512bw");
52 }
53 #[cfg(target_arch = "aarch64")]
54 {
55 capabilities.neon = std::arch::is_aarch64_feature_detected!("neon");
56 capabilities.sve = std::arch::is_aarch64_feature_detected!("sve");
57 capabilities.sme = detect_sme();
58 }
59 capabilities
60}
61
62#[cfg(all(
63 target_arch = "aarch64",
64 any(target_os = "linux", target_os = "android")
65))]
66fn detect_sme() -> bool {
67 use std::ffi::c_ulong;
68
69 const AT_HWCAP2: c_ulong = 26;
70 const HWCAP2_SME: c_ulong = 1 << 23;
71
72 unsafe extern "C" {
73 fn getauxval(kind: c_ulong) -> c_ulong;
74 }
75
76 unsafe { getauxval(AT_HWCAP2) & HWCAP2_SME != 0 }
79}
80
81#[cfg(all(
82 target_arch = "aarch64",
83 not(any(target_os = "linux", target_os = "android"))
84))]
85const fn detect_sme() -> bool {
86 false
87}
88
89pub fn simd_backend() -> SimdBackend {
95 static BACKEND: std::sync::OnceLock<SimdBackend> = std::sync::OnceLock::new();
97 *BACKEND.get_or_init(|| {
98 let capabilities = hardware_capabilities();
99 if capabilities.avx2 {
100 SimdBackend::Avx2
101 } else if capabilities.sse2 {
102 SimdBackend::Sse2
103 } else if capabilities.neon {
104 SimdBackend::Neon
105 } else {
106 SimdBackend::Scalar
107 }
108 })
109}
110
111#[cfg(feature = "adaptive")]
112pub(crate) fn is_ascii(bytes: &[u8]) -> bool {
113 first_non_ascii(bytes) == bytes.len()
114}
115
116#[cfg(feature = "adaptive")]
117pub(crate) fn plain_varint_prefix(bytes: &[u8]) -> usize {
118 match simd_backend() {
119 #[cfg(target_arch = "x86_64")]
120 SimdBackend::Avx2 => {
121 unsafe { x86::plain_prefix_avx2(bytes) }
124 }
125 #[cfg(target_arch = "x86_64")]
126 SimdBackend::Sse2 => {
127 unsafe { x86::plain_prefix_sse2(bytes) }
129 }
130 #[cfg(target_arch = "aarch64")]
131 SimdBackend::Neon => {
132 unsafe { arm::plain_prefix_neon(bytes) }
134 }
135 _ => scalar_plain_prefix(bytes),
136 }
137}
138
139#[cfg(feature = "adaptive")]
140fn first_non_ascii(bytes: &[u8]) -> usize {
141 match simd_backend() {
142 #[cfg(target_arch = "x86_64")]
143 SimdBackend::Avx2 => {
144 unsafe { x86::ascii_prefix_avx2(bytes) }
146 }
147 #[cfg(target_arch = "x86_64")]
148 SimdBackend::Sse2 => {
149 unsafe { x86::ascii_prefix_sse2(bytes) }
151 }
152 #[cfg(target_arch = "aarch64")]
153 SimdBackend::Neon => {
154 unsafe { arm::ascii_prefix_neon(bytes) }
156 }
157 _ => scalar_ascii_prefix(bytes),
158 }
159}
160
161#[cfg(feature = "adaptive")]
162fn scalar_ascii_prefix(bytes: &[u8]) -> usize {
163 bytes
164 .iter()
165 .position(|byte| !byte.is_ascii())
166 .unwrap_or(bytes.len())
167}
168
169#[cfg(feature = "adaptive")]
170fn scalar_plain_prefix(bytes: &[u8]) -> usize {
171 bytes
172 .iter()
173 .position(|byte| *byte > 250)
174 .unwrap_or(bytes.len())
175}
176
177#[cfg(all(feature = "adaptive", target_arch = "x86_64"))]
178mod x86 {
179 #![allow(unused_unsafe)]
184 use std::arch::x86_64::*;
185
186 #[target_feature(enable = "avx2")]
187 pub(super) unsafe fn ascii_prefix_avx2(bytes: &[u8]) -> usize {
188 let mut offset = 0;
189 while offset + 32 <= bytes.len() {
190 let vector = unsafe { _mm256_loadu_si256(bytes.as_ptr().add(offset).cast()) };
192 let mask = unsafe { _mm256_movemask_epi8(vector) } as u32;
193 if mask != 0 {
194 return offset + mask.trailing_zeros() as usize;
195 }
196 offset += 32;
197 }
198 offset + super::scalar_ascii_prefix(&bytes[offset..])
199 }
200
201 #[target_feature(enable = "sse2")]
202 pub(super) unsafe fn ascii_prefix_sse2(bytes: &[u8]) -> usize {
203 let mut offset = 0;
204 while offset + 16 <= bytes.len() {
205 let vector = unsafe { _mm_loadu_si128(bytes.as_ptr().add(offset).cast()) };
207 let mask = unsafe { _mm_movemask_epi8(vector) } as u32;
208 if mask != 0 {
209 return offset + mask.trailing_zeros() as usize;
210 }
211 offset += 16;
212 }
213 offset + super::scalar_ascii_prefix(&bytes[offset..])
214 }
215
216 #[target_feature(enable = "avx2")]
217 pub(super) unsafe fn plain_prefix_avx2(bytes: &[u8]) -> usize {
218 let mut offset = 0;
219 let sign = unsafe { _mm256_set1_epi8(i8::MIN) };
220 let threshold = unsafe { _mm256_set1_epi8(122) };
221 while offset + 32 <= bytes.len() {
222 let vector = unsafe { _mm256_loadu_si256(bytes.as_ptr().add(offset).cast()) };
224 let unsigned_order = unsafe { _mm256_xor_si256(vector, sign) };
225 let markers = unsafe { _mm256_cmpgt_epi8(unsigned_order, threshold) };
226 let mask = unsafe { _mm256_movemask_epi8(markers) } as u32;
227 if mask != 0 {
228 return offset + mask.trailing_zeros() as usize;
229 }
230 offset += 32;
231 }
232 offset + super::scalar_plain_prefix(&bytes[offset..])
233 }
234
235 #[target_feature(enable = "sse2")]
236 pub(super) unsafe fn plain_prefix_sse2(bytes: &[u8]) -> usize {
237 let mut offset = 0;
238 let sign = unsafe { _mm_set1_epi8(i8::MIN) };
239 let threshold = unsafe { _mm_set1_epi8(122) };
240 while offset + 16 <= bytes.len() {
241 let vector = unsafe { _mm_loadu_si128(bytes.as_ptr().add(offset).cast()) };
243 let unsigned_order = unsafe { _mm_xor_si128(vector, sign) };
244 let markers = unsafe { _mm_cmpgt_epi8(unsigned_order, threshold) };
245 let mask = unsafe { _mm_movemask_epi8(markers) } as u32;
246 if mask != 0 {
247 return offset + mask.trailing_zeros() as usize;
248 }
249 offset += 16;
250 }
251 offset + super::scalar_plain_prefix(&bytes[offset..])
252 }
253}
254
255#[cfg(all(feature = "adaptive", target_arch = "aarch64"))]
256mod arm {
257 #![allow(unused_unsafe)]
260 use std::arch::aarch64::*;
261
262 #[target_feature(enable = "neon")]
263 pub(super) unsafe fn ascii_prefix_neon(bytes: &[u8]) -> usize {
264 let mut offset = 0;
265 let threshold = unsafe { vdupq_n_u8(127) };
266 while offset + 16 <= bytes.len() {
267 let vector = unsafe { vld1q_u8(bytes.as_ptr().add(offset)) };
269 let compared = unsafe { vcgtq_u8(vector, threshold) };
270 let mut lanes = [0u8; 16];
271 unsafe { vst1q_u8(lanes.as_mut_ptr(), compared) };
273 if let Some(index) = lanes.iter().position(|lane| *lane != 0) {
274 return offset + index;
275 }
276 offset += 16;
277 }
278 offset + super::scalar_ascii_prefix(&bytes[offset..])
279 }
280
281 #[target_feature(enable = "neon")]
282 pub(super) unsafe fn plain_prefix_neon(bytes: &[u8]) -> usize {
283 let mut offset = 0;
284 let threshold = unsafe { vdupq_n_u8(250) };
285 while offset + 16 <= bytes.len() {
286 let vector = unsafe { vld1q_u8(bytes.as_ptr().add(offset)) };
288 let compared = unsafe { vcgtq_u8(vector, threshold) };
289 let mut lanes = [0u8; 16];
290 unsafe { vst1q_u8(lanes.as_mut_ptr(), compared) };
292 if let Some(index) = lanes.iter().position(|lane| *lane != 0) {
293 return offset + index;
294 }
295 offset += 16;
296 }
297 offset + super::scalar_plain_prefix(&bytes[offset..])
298 }
299}
300
301#[cfg(all(test, feature = "adaptive"))]
302mod tests {
303 use super::*;
304 use alloc::vec;
305 use alloc::vec::Vec;
306
307 #[test]
308 fn dispatched_scanners_match_scalar_at_vector_boundaries() {
309 for length in 0..96 {
310 let ascii = vec![b'a'; length];
311 assert_eq!(first_non_ascii(&ascii), scalar_ascii_prefix(&ascii));
312 assert_eq!(plain_varint_prefix(&ascii), scalar_plain_prefix(&ascii));
313
314 for position in 0..length {
315 let mut non_ascii = ascii.clone();
316 non_ascii[position] = 0x80;
317 assert_eq!(first_non_ascii(&non_ascii), position);
318
319 let mut marker = ascii.clone();
320 marker[position] = 251;
321 assert_eq!(plain_varint_prefix(&marker), position);
322 }
323 }
324 }
325
326 #[test]
327 fn plain_varint_scan_accepts_entire_single_byte_domain() {
328 let values = (0..=250).map(|value| value as u8).collect::<Vec<_>>();
329 assert_eq!(plain_varint_prefix(&values), values.len());
330 for marker in 251..=255 {
331 let mut input = values.clone();
332 input.push(marker);
333 assert_eq!(plain_varint_prefix(&input), values.len());
334 }
335 }
336}