simd_json/
numberparse.rs

1#[cfg(not(feature = "approx-number-parsing"))]
2mod correct;
3
4#[cfg(feature = "approx-number-parsing")]
5mod approx;
6
7use crate::safer_unchecked::GetSaferUnchecked;
8
9#[cfg(all(target_arch = "x86", feature = "swar-number-parsing"))]
10use std::arch::x86 as arch;
11
12#[cfg(all(target_arch = "x86_64", feature = "swar-number-parsing"))]
13use std::arch::x86_64 as arch;
14
15#[cfg(all(
16    any(target_arch = "x86", target_arch = "x86_64"),
17    feature = "swar-number-parsing"
18))]
19use arch::{
20    __m128i, _mm_cvtsi128_si32, _mm_loadu_si128, _mm_madd_epi16, _mm_maddubs_epi16,
21    _mm_packus_epi32, _mm_set1_epi8, _mm_setr_epi8, _mm_setr_epi16, _mm_sub_epi8,
22};
23
24#[cfg_attr(not(feature = "no-inline"), inline)]
25pub fn is_integer(c: u8) -> bool {
26    c.is_ascii_digit()
27}
28
29// We need to check that the character following a zero is valid. This is
30// probably frequent and it is hard than it looks. We are building all of this
31// just to differentiate between 0x1 (invalid), 0,1 (valid) 0e1 (valid)...
32const STRUCTURAL_OR_WHITESPACE_OR_EXPONENT_OR_DECIMAL_NEGATED: [bool; 256] = [
33    false, true, true, true, true, true, true, true, true, false, false, true, true, false, true,
34    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
35    true, false, true, true, true, true, true, true, true, true, true, true, true, false, true,
36    false, true, true, true, true, true, true, true, true, true, true, true, false, true, true,
37    true, true, true, true, true, true, true, true, false, true, true, true, true, true, true,
38    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
39    false, true, false, true, true, true, true, true, true, true, false, true, true, true, true,
40    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
41    true, false, true, false, true, true, true, true, true, true, true, true, true, true, true,
42    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
43    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
44    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
45    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
46    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
47    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
48    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
49    true, true, true, true, true, true, true,
50];
51
52#[cfg_attr(not(feature = "no-inline"), inline)]
53fn is_not_structural_or_whitespace_or_exponent_or_decimal(c: u8) -> bool {
54    unsafe {
55        *STRUCTURAL_OR_WHITESPACE_OR_EXPONENT_OR_DECIMAL_NEGATED.get_kinda_unchecked(c as usize)
56    }
57}
58
59// #ifdef _MSC_VER
60// check quickly whether the next 8 chars are made of digits
61// at a glance, it looks better than Mula's
62// http://0x80.pl/articles/swar-digits-validate.html
63
64#[cfg(feature = "swar-number-parsing")]
65#[cfg_attr(not(feature = "no-inline"), inline)]
66#[allow(clippy::cast_ptr_alignment)]
67fn is_made_of_eight_digits_fast(chars: [u8; 8]) -> bool {
68    let val = u64::from_ne_bytes(chars);
69
70    ((val & 0xF0F0_F0F0_F0F0_F0F0)
71        | (((val.wrapping_add(0x0606_0606_0606_0606)) & 0xF0F0_F0F0_F0F0_F0F0) >> 4))
72        == 0x3333_3333_3333_3333
73}
74
75#[cfg_attr(not(feature = "no-inline"), inline)]
76#[cfg(all(
77    any(target_arch = "x86", target_arch = "x86_64"),
78    feature = "swar-number-parsing"
79))]
80#[allow(
81    clippy::cast_sign_loss,
82    clippy::cast_possible_wrap,
83    clippy::cast_ptr_alignment
84)]
85fn parse_eight_digits_unrolled(chars: &[u8]) -> u32 {
86    unsafe {
87        // this actually computes *16* values so we are being wasteful.
88        let ascii0: __m128i = _mm_set1_epi8(b'0' as i8);
89        let mul_1_10: __m128i =
90            _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1);
91        let mul_1_100: __m128i = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1);
92        let mul_1_10000: __m128i = _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1);
93        // We know what we're doing right? :P
94        let input: __m128i = _mm_sub_epi8(
95            _mm_loadu_si128(
96                chars
97                    .get_kinda_unchecked(0..16)
98                    .as_ptr()
99                    .cast::<arch::__m128i>(),
100            ),
101            ascii0,
102        );
103        let t1: __m128i = _mm_maddubs_epi16(input, mul_1_10);
104        let t2: __m128i = _mm_madd_epi16(t1, mul_1_100);
105        let t3: __m128i = _mm_packus_epi32(t2, t2);
106        let t4: __m128i = _mm_madd_epi16(t3, mul_1_10000);
107        _mm_cvtsi128_si32(t4) as u32 // only captures the sum of the first 8 digits, drop the rest
108    }
109}
110
111#[cfg_attr(not(feature = "no-inline"), inline)]
112#[cfg(all(
113    not(any(target_arch = "x86", target_arch = "x86_64")),
114    feature = "swar-number-parsing"
115))]
116#[allow(clippy::cast_ptr_alignment)]
117fn parse_eight_digits_unrolled(chars: &[u8]) -> u32 {
118    let val = unsafe { chars.as_ptr().cast::<u64>().read_unaligned() }; //    memcpy(&val, chars, sizeof(u64));
119    let val = (val & 0x0F0F_0F0F_0F0F_0F0F).wrapping_mul(2561) >> 8;
120    let val = (val & 0x00FF_00FF_00FF_00FF).wrapping_mul(6_553_601) >> 16;
121
122    ((val & 0x0000_FFFF_0000_FFFF).wrapping_mul(42_949_672_960_001) >> 32) as u32
123}
simd_json/numberparse.rs

simd_json/
numberparse.rs
