Skip to main content

simdutf8/implementation/x86/
avx2.rs

1//! Contains the x86-64/x86 AVX2 UTF-8 validation implementation.
2
3#[cfg(target_arch = "x86")]
4use core::arch::x86::{
5    __m256i, _mm256_alignr_epi8, _mm256_and_si256, _mm256_loadu_si256, _mm256_movemask_epi8,
6    _mm256_or_si256, _mm256_permute2x128_si256, _mm256_set1_epi8, _mm256_setr_epi8,
7    _mm256_setzero_si256, _mm256_shuffle_epi8, _mm256_srli_epi16, _mm256_subs_epu8,
8    _mm256_testz_si256, _mm256_xor_si256, _mm_prefetch, _MM_HINT_T0,
9};
10#[cfg(target_arch = "x86_64")]
11use core::arch::x86_64::{
12    __m256i, _mm256_alignr_epi8, _mm256_and_si256, _mm256_loadu_si256, _mm256_movemask_epi8,
13    _mm256_or_si256, _mm256_permute2x128_si256, _mm256_set1_epi8, _mm256_setr_epi8,
14    _mm256_setzero_si256, _mm256_shuffle_epi8, _mm256_srli_epi16, _mm256_subs_epu8,
15    _mm256_testz_si256, _mm256_xor_si256, _mm_prefetch, _MM_HINT_T0,
16};
17
18use crate::implementation::helpers::Utf8CheckAlgorithm;
19
20// AVX 2 SIMD primitives
21
22type SimdU8Value = crate::implementation::helpers::SimdU8Value<__m256i>;
23
24impl SimdU8Value {
25    #[allow(clippy::cast_possible_wrap)]
26    #[allow(clippy::too_many_arguments)]
27    #[target_feature(enable = "avx2")]
28    #[inline]
29    unsafe fn from_32_cut_off_leading(
30        v0: u8,
31        v1: u8,
32        v2: u8,
33        v3: u8,
34        v4: u8,
35        v5: u8,
36        v6: u8,
37        v7: u8,
38        v8: u8,
39        v9: u8,
40        v10: u8,
41        v11: u8,
42        v12: u8,
43        v13: u8,
44        v14: u8,
45        v15: u8,
46        v16: u8,
47        v17: u8,
48        v18: u8,
49        v19: u8,
50        v20: u8,
51        v21: u8,
52        v22: u8,
53        v23: u8,
54        v24: u8,
55        v25: u8,
56        v26: u8,
57        v27: u8,
58        v28: u8,
59        v29: u8,
60        v30: u8,
61        v31: u8,
62    ) -> Self {
63        Self::from(_mm256_setr_epi8(
64            v0 as i8, v1 as i8, v2 as i8, v3 as i8, v4 as i8, v5 as i8, v6 as i8, v7 as i8,
65            v8 as i8, v9 as i8, v10 as i8, v11 as i8, v12 as i8, v13 as i8, v14 as i8, v15 as i8,
66            v16 as i8, v17 as i8, v18 as i8, v19 as i8, v20 as i8, v21 as i8, v22 as i8, v23 as i8,
67            v24 as i8, v25 as i8, v26 as i8, v27 as i8, v28 as i8, v29 as i8, v30 as i8, v31 as i8,
68        ))
69    }
70
71    #[allow(clippy::too_many_arguments)]
72    #[target_feature(enable = "avx2")]
73    #[inline]
74    unsafe fn repeat_16(
75        v0: u8,
76        v1: u8,
77        v2: u8,
78        v3: u8,
79        v4: u8,
80        v5: u8,
81        v6: u8,
82        v7: u8,
83        v8: u8,
84        v9: u8,
85        v10: u8,
86        v11: u8,
87        v12: u8,
88        v13: u8,
89        v14: u8,
90        v15: u8,
91    ) -> Self {
92        Self::from_32_cut_off_leading(
93            v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v0, v1, v2, v3,
94            v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15,
95        )
96    }
97
98    #[allow(clippy::cast_ptr_alignment)]
99    #[target_feature(enable = "avx2")]
100    #[inline]
101    unsafe fn load_from(ptr: *const u8) -> Self {
102        Self::from(_mm256_loadu_si256(ptr.cast::<__m256i>()))
103    }
104
105    #[target_feature(enable = "avx2")]
106    #[inline]
107    unsafe fn lookup_16(self, tbl: Self) -> Self {
108        Self::from(_mm256_shuffle_epi8(tbl.0, self.0))
109    }
110
111    #[allow(clippy::cast_possible_wrap)]
112    #[target_feature(enable = "avx2")]
113    #[inline]
114    unsafe fn splat(val: u8) -> Self {
115        Self::from(_mm256_set1_epi8(val as i8))
116    }
117
118    #[target_feature(enable = "avx2")]
119    #[inline]
120    unsafe fn splat0() -> Self {
121        Self::from(_mm256_setzero_si256())
122    }
123
124    #[target_feature(enable = "avx2")]
125    #[inline]
126    unsafe fn or(self, b: Self) -> Self {
127        Self::from(_mm256_or_si256(self.0, b.0))
128    }
129
130    #[target_feature(enable = "avx2")]
131    #[inline]
132    unsafe fn and(self, b: Self) -> Self {
133        Self::from(_mm256_and_si256(self.0, b.0))
134    }
135
136    #[target_feature(enable = "avx2")]
137    #[inline]
138    unsafe fn xor(self, b: Self) -> Self {
139        Self::from(_mm256_xor_si256(self.0, b.0))
140    }
141
142    #[target_feature(enable = "avx2")]
143    #[inline]
144    unsafe fn saturating_sub(self, b: Self) -> Self {
145        Self::from(_mm256_subs_epu8(self.0, b.0))
146    }
147
148    // ugly but shr<N> requires const generics
149    #[target_feature(enable = "avx2")]
150    #[inline]
151    unsafe fn shr4(self) -> Self {
152        Self::from(_mm256_srli_epi16(self.0, 4)).and(Self::splat(0xFF >> 4))
153    }
154
155    // ugly but prev<N> requires const generics
156    #[target_feature(enable = "avx2")]
157    #[inline]
158    unsafe fn prev1(self, prev: Self) -> Self {
159        Self::from(_mm256_alignr_epi8(
160            self.0,
161            _mm256_permute2x128_si256(prev.0, self.0, 0x21),
162            16 - 1,
163        ))
164    }
165
166    // ugly but prev<N> requires const generics
167    #[target_feature(enable = "avx2")]
168    #[inline]
169    unsafe fn prev2(self, prev: Self) -> Self {
170        Self::from(_mm256_alignr_epi8(
171            self.0,
172            _mm256_permute2x128_si256(prev.0, self.0, 0x21),
173            16 - 2,
174        ))
175    }
176
177    // ugly but prev<N> requires const generics
178    #[target_feature(enable = "avx2")]
179    #[inline]
180    unsafe fn prev3(self, prev: Self) -> Self {
181        Self::from(_mm256_alignr_epi8(
182            self.0,
183            _mm256_permute2x128_si256(prev.0, self.0, 0x21),
184            16 - 3,
185        ))
186    }
187
188    #[target_feature(enable = "avx2")]
189    #[inline]
190    unsafe fn any_bit_set(self) -> bool {
191        _mm256_testz_si256(self.0, self.0) != 1
192    }
193
194    #[target_feature(enable = "avx2")]
195    #[inline]
196    unsafe fn is_ascii(self) -> bool {
197        _mm256_movemask_epi8(self.0) == 0
198    }
199}
200
201impl From<__m256i> for SimdU8Value {
202    #[inline]
203    fn from(val: __m256i) -> Self {
204        Self(val)
205    }
206}
207
208#[target_feature(enable = "avx2")]
209#[inline]
210unsafe fn simd_prefetch(ptr: *const u8) {
211    _mm_prefetch(ptr.cast::<i8>(), _MM_HINT_T0);
212}
213
214const PREFETCH: bool = true;
215use crate::implementation::helpers::TempSimdChunkA32 as TempSimdChunk;
216simd_input_256_bit!(#[target_feature(enable = "avx2")]);
217algorithm_simd!(#[target_feature(enable = "avx2")]);
218algorithm_simd_default_special_case_fns!(#[target_feature(enable = "avx2")]);