Skip to main content

simdutf8/implementation/x86/
sse42.rs

1//! Contains the x86-64/x86 SSE4.2 UTF-8 validation implementation.
2
3#[cfg(target_arch = "x86")]
4use core::arch::x86::{
5    __m128i, _mm_alignr_epi8, _mm_and_si128, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128,
6    _mm_prefetch, _mm_set1_epi8, _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8,
7    _mm_srli_epi16, _mm_subs_epu8, _mm_testz_si128, _mm_xor_si128, _MM_HINT_T0,
8};
9#[cfg(target_arch = "x86_64")]
10use core::arch::x86_64::{
11    __m128i, _mm_alignr_epi8, _mm_and_si128, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128,
12    _mm_prefetch, _mm_set1_epi8, _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8,
13    _mm_srli_epi16, _mm_subs_epu8, _mm_testz_si128, _mm_xor_si128, _MM_HINT_T0,
14};
15
16use crate::implementation::helpers::Utf8CheckAlgorithm;
17
18// SSE 4.2 SIMD primitives
19
20type SimdU8Value = crate::implementation::helpers::SimdU8Value<__m128i>;
21
22impl SimdU8Value {
23    #[allow(clippy::too_many_arguments)]
24    #[allow(clippy::cast_possible_wrap)]
25    #[target_feature(enable = "sse4.2")]
26    #[inline]
27    unsafe fn from_32_cut_off_leading(
28        _v0: u8,
29        _v1: u8,
30        _v2: u8,
31        _v3: u8,
32        _v4: u8,
33        _v5: u8,
34        _v6: u8,
35        _v7: u8,
36        _v8: u8,
37        _v9: u8,
38        _v10: u8,
39        _v11: u8,
40        _v12: u8,
41        _v13: u8,
42        _v14: u8,
43        _v15: u8,
44        v16: u8,
45        v17: u8,
46        v18: u8,
47        v19: u8,
48        v20: u8,
49        v21: u8,
50        v22: u8,
51        v23: u8,
52        v24: u8,
53        v25: u8,
54        v26: u8,
55        v27: u8,
56        v28: u8,
57        v29: u8,
58        v30: u8,
59        v31: u8,
60    ) -> Self {
61        Self::from(_mm_setr_epi8(
62            v16 as i8, v17 as i8, v18 as i8, v19 as i8, v20 as i8, v21 as i8, v22 as i8, v23 as i8,
63            v24 as i8, v25 as i8, v26 as i8, v27 as i8, v28 as i8, v29 as i8, v30 as i8, v31 as i8,
64        ))
65    }
66
67    #[allow(clippy::too_many_arguments)]
68    #[allow(clippy::cast_possible_wrap)]
69    #[target_feature(enable = "sse4.2")]
70    #[inline]
71    unsafe fn repeat_16(
72        v0: u8,
73        v1: u8,
74        v2: u8,
75        v3: u8,
76        v4: u8,
77        v5: u8,
78        v6: u8,
79        v7: u8,
80        v8: u8,
81        v9: u8,
82        v10: u8,
83        v11: u8,
84        v12: u8,
85        v13: u8,
86        v14: u8,
87        v15: u8,
88    ) -> Self {
89        Self::from(_mm_setr_epi8(
90            v0 as i8, v1 as i8, v2 as i8, v3 as i8, v4 as i8, v5 as i8, v6 as i8, v7 as i8,
91            v8 as i8, v9 as i8, v10 as i8, v11 as i8, v12 as i8, v13 as i8, v14 as i8, v15 as i8,
92        ))
93    }
94
95    #[allow(clippy::cast_ptr_alignment)]
96    #[target_feature(enable = "sse4.2")]
97    #[inline]
98    unsafe fn load_from(ptr: *const u8) -> Self {
99        Self::from(_mm_loadu_si128(ptr.cast::<__m128i>()))
100    }
101
102    #[target_feature(enable = "sse4.2")]
103    #[inline]
104    unsafe fn lookup_16(self, tbl: Self) -> Self {
105        Self::from(_mm_shuffle_epi8(tbl.0, self.0))
106    }
107
108    #[allow(clippy::cast_possible_wrap)]
109    #[target_feature(enable = "sse4.2")]
110    #[inline]
111    unsafe fn splat(val: u8) -> Self {
112        Self::from(_mm_set1_epi8(val as i8))
113    }
114
115    #[target_feature(enable = "sse4.2")]
116    #[inline]
117    unsafe fn splat0() -> Self {
118        Self::from(_mm_setzero_si128())
119    }
120
121    #[target_feature(enable = "sse4.2")]
122    #[inline]
123    unsafe fn or(self, b: Self) -> Self {
124        Self::from(_mm_or_si128(self.0, b.0))
125    }
126
127    #[target_feature(enable = "sse4.2")]
128    #[inline]
129    unsafe fn and(self, b: Self) -> Self {
130        Self::from(_mm_and_si128(self.0, b.0))
131    }
132
133    #[target_feature(enable = "sse4.2")]
134    #[inline]
135    unsafe fn xor(self, b: Self) -> Self {
136        Self::from(_mm_xor_si128(self.0, b.0))
137    }
138
139    #[target_feature(enable = "sse4.2")]
140    #[inline]
141    unsafe fn saturating_sub(self, b: Self) -> Self {
142        Self::from(_mm_subs_epu8(self.0, b.0))
143    }
144
145    // ugly but shr<N> requires const generics
146    #[target_feature(enable = "sse4.2")]
147    #[inline]
148    unsafe fn shr4(self) -> Self {
149        Self::from(_mm_srli_epi16(self.0, 4)).and(Self::splat(0xFF >> 4))
150    }
151
152    // ugly but prev<N> requires const generics
153    #[target_feature(enable = "sse4.2")]
154    #[inline]
155    unsafe fn prev1(self, prev: Self) -> Self {
156        Self::from(_mm_alignr_epi8(self.0, prev.0, 16 - 1))
157    }
158
159    // ugly but prev<N> requires const generics
160    #[target_feature(enable = "sse4.2")]
161    #[inline]
162    unsafe fn prev2(self, prev: Self) -> Self {
163        Self::from(_mm_alignr_epi8(self.0, prev.0, 16 - 2))
164    }
165
166    // ugly but prev<N> requires const generics
167    #[target_feature(enable = "sse4.2")]
168    #[inline]
169    unsafe fn prev3(self, prev: Self) -> Self {
170        Self::from(_mm_alignr_epi8(self.0, prev.0, 16 - 3))
171    }
172
173    #[target_feature(enable = "sse4.2")]
174    #[inline]
175    unsafe fn any_bit_set(self) -> bool {
176        _mm_testz_si128(self.0, self.0) != 1
177    }
178
179    #[target_feature(enable = "sse4.2")]
180    #[inline]
181    unsafe fn is_ascii(self) -> bool {
182        _mm_movemask_epi8(self.0) == 0
183    }
184}
185
186impl From<__m128i> for SimdU8Value {
187    #[inline]
188    fn from(val: __m128i) -> Self {
189        Self(val)
190    }
191}
192
193#[target_feature(enable = "sse4.2")]
194#[inline]
195unsafe fn simd_prefetch(ptr: *const u8) {
196    _mm_prefetch(ptr.cast::<i8>(), _MM_HINT_T0);
197}
198
199const PREFETCH: bool = false;
200use crate::implementation::helpers::TempSimdChunkA16 as TempSimdChunk;
201simd_input_128_bit!(#[target_feature(enable = "sse4.2")]);
202algorithm_simd!(#[target_feature(enable = "sse4.2")]);
203algorithm_simd_default_special_case_fns!(#[target_feature(enable = "sse4.2")]);