simd_json/
numberparse.rs

1#[cfg(not(feature = "approx-number-parsing"))]
2mod correct;
3
4#[cfg(feature = "approx-number-parsing")]
5mod approx;
6
7use crate::safer_unchecked::GetSaferUnchecked;
8
9#[cfg(all(
10    target_arch = "x86",
11    target_feature = "ssse3",
12    feature = "swar-number-parsing"
13))]
14use std::arch::x86 as arch;
15
16#[cfg(all(
17    target_arch = "x86_64",
18    target_feature = "ssse3",
19    feature = "swar-number-parsing"
20))]
21use std::arch::x86_64 as arch;
22
23#[cfg(all(
24    any(target_arch = "x86", target_arch = "x86_64"),
25    target_feature = "ssse3",
26    feature = "swar-number-parsing"
27))]
28use arch::{
29    __m128i, _mm_cvtsi128_si32, _mm_loadu_si128, _mm_madd_epi16, _mm_maddubs_epi16,
30    _mm_packus_epi32, _mm_set1_epi8, _mm_setr_epi8, _mm_setr_epi16, _mm_sub_epi8,
31};
32
33#[cfg_attr(not(feature = "no-inline"), inline)]
34pub fn is_integer(c: u8) -> bool {
35    c.is_ascii_digit()
36}
37
38// We need to check that the character following a zero is valid. This is
39// probably frequent and it is hard than it looks. We are building all of this
40// just to differentiate between 0x1 (invalid), 0,1 (valid) 0e1 (valid)...
41const STRUCTURAL_OR_WHITESPACE_OR_EXPONENT_OR_DECIMAL_NEGATED: [bool; 256] = [
42    false, true, true, true, true, true, true, true, true, false, false, true, true, false, true,
43    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
44    true, false, true, true, true, true, true, true, true, true, true, true, true, false, true,
45    false, true, true, true, true, true, true, true, true, true, true, true, false, true, true,
46    true, true, true, true, true, true, true, true, false, true, true, true, true, true, true,
47    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
48    false, true, false, true, true, true, true, true, true, true, false, true, true, true, true,
49    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
50    true, false, true, false, true, true, true, true, true, true, true, true, true, true, true,
51    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
52    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
53    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
54    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
55    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
56    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
57    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
58    true, true, true, true, true, true, true,
59];
60
61#[cfg_attr(not(feature = "no-inline"), inline)]
62fn is_not_structural_or_whitespace_or_exponent_or_decimal(c: u8) -> bool {
63    unsafe {
64        *STRUCTURAL_OR_WHITESPACE_OR_EXPONENT_OR_DECIMAL_NEGATED.get_kinda_unchecked(c as usize)
65    }
66}
67
68// #ifdef _MSC_VER
69// check quickly whether the next 8 chars are made of digits
70// at a glance, it looks better than Mula's
71// http://0x80.pl/articles/swar-digits-validate.html
72
73#[cfg(feature = "swar-number-parsing")]
74#[cfg_attr(not(feature = "no-inline"), inline)]
75#[allow(clippy::cast_ptr_alignment)]
76fn is_made_of_eight_digits_fast(chars: [u8; 8]) -> bool {
77    let val = u64::from_ne_bytes(chars);
78
79    ((val & 0xF0F0_F0F0_F0F0_F0F0)
80        | (((val.wrapping_add(0x0606_0606_0606_0606)) & 0xF0F0_F0F0_F0F0_F0F0) >> 4))
81        == 0x3333_3333_3333_3333
82}
83
84#[cfg_attr(not(feature = "no-inline"), inline)]
85#[cfg(all(
86    any(target_arch = "x86", target_arch = "x86_64"),
87    target_feature = "ssse3",
88    feature = "swar-number-parsing",
89))]
90#[target_feature(enable = "ssse3")]
91#[allow(
92    clippy::cast_sign_loss,
93    clippy::cast_possible_wrap,
94    clippy::cast_ptr_alignment
95)]
96unsafe fn parse_eight_digits_ssse3(chars: &[u8]) -> u32 {
97    unsafe {
98        // this actually computes *16* values so we are being wasteful.
99        let ascii0: __m128i = _mm_set1_epi8(b'0' as i8);
100        let mul_1_10: __m128i =
101            _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1);
102        let mul_1_100: __m128i = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1);
103        let mul_1_10000: __m128i = _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1);
104        // We know what we're doing right? :P
105        let input: __m128i = _mm_sub_epi8(
106            _mm_loadu_si128(
107                chars
108                    .get_kinda_unchecked(0..16)
109                    .as_ptr()
110                    .cast::<arch::__m128i>(),
111            ),
112            ascii0,
113        );
114        let t1: __m128i = _mm_maddubs_epi16(input, mul_1_10);
115        let t2: __m128i = _mm_madd_epi16(t1, mul_1_100);
116        let t3: __m128i = _mm_packus_epi32(t2, t2);
117        let t4: __m128i = _mm_madd_epi16(t3, mul_1_10000);
118        _mm_cvtsi128_si32(t4) as u32 // only captures the sum of the first 8 digits, drop the rest
119    }
120}
121
122#[cfg_attr(not(feature = "no-inline"), inline)]
123#[cfg(feature = "swar-number-parsing")]
124#[allow(clippy::cast_ptr_alignment)]
125fn parse_eight_digits_swar(chars: &[u8]) -> u32 {
126    // The SWAR reduction below assumes the eight digit bytes are laid out
127    // least-significant-first, so interpret the input little-endian on every
128    // target (a no-op on little-endian, a byte swap on big-endian like s390x).
129    let val = u64::from_le(unsafe { chars.as_ptr().cast::<u64>().read_unaligned() });
130    let val = (val & 0x0F0F_0F0F_0F0F_0F0F).wrapping_mul(2561) >> 8;
131    let val = (val & 0x00FF_00FF_00FF_00FF).wrapping_mul(6_553_601) >> 16;
132
133    ((val & 0x0000_FFFF_0000_FFFF).wrapping_mul(42_949_672_960_001) >> 32) as u32
134}
135
136#[cfg(feature = "swar-number-parsing")]
137#[cfg_attr(not(feature = "no-inline"), inline)]
138fn parse_eight_digits_unrolled(chars: &[u8]) -> u32 {
139    // Only use the SSSE3 version when the feature is statically available:
140    // a per-call `is_x86_feature_detected!` check costs more than the SWAR
141    // fallback (cached-atomic load + two outlined `target_feature` calls per
142    // 8-digit group).
143    #[cfg(all(
144        any(target_arch = "x86", target_arch = "x86_64"),
145        target_feature = "ssse3"
146    ))]
147    {
148        return unsafe { parse_eight_digits_ssse3(chars) };
149    }
150    #[allow(unreachable_code)]
151    parse_eight_digits_swar(chars)
152}
simd_json/numberparse.rs

simd_json/
numberparse.rs
