1#[cfg(target_arch = "x86")]
4use core::arch::x86::{
5 __m512i, _mm512_alignr_epi8, _mm512_loadu_si512, _mm512_maskz_loadu_epi8, _mm512_movepi8_mask,
6 _mm512_or_si512, _mm512_permutex2var_epi64, _mm512_permutexvar_epi8, _mm512_set1_epi8,
7 _mm512_set_epi64, _mm512_set_epi8, _mm512_setzero_si512, _mm512_srli_epi16, _mm512_subs_epu8,
8 _mm512_ternarylogic_epi32, _mm512_test_epi8_mask, _mm_prefetch, _MM_HINT_T0,
9};
10
11#[cfg(target_arch = "x86_64")]
12use core::arch::x86_64::{
13 __m512i, _mm512_alignr_epi8, _mm512_loadu_si512, _mm512_maskz_loadu_epi8, _mm512_movepi8_mask,
14 _mm512_or_si512, _mm512_permutex2var_epi64, _mm512_permutexvar_epi8, _mm512_set1_epi8,
15 _mm512_set_epi64, _mm512_set_epi8, _mm512_setzero_si512, _mm512_srli_epi16, _mm512_subs_epu8,
16 _mm512_ternarylogic_epi32, _mm512_test_epi8_mask, _mm_prefetch, _MM_HINT_T0,
17};
18
19use crate::implementation::helpers::Utf8CheckAlgorithm;
20
21type SimdU8Value = crate::implementation::helpers::SimdU8Value<__m512i>;
24
25impl SimdU8Value {
26 #[allow(clippy::cast_possible_wrap)]
27 #[allow(clippy::too_many_arguments)]
28 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
29 #[inline]
30 unsafe fn from_32_cut_off_leading(
31 v0: u8,
32 v1: u8,
33 v2: u8,
34 v3: u8,
35 v4: u8,
36 v5: u8,
37 v6: u8,
38 v7: u8,
39 v8: u8,
40 v9: u8,
41 v10: u8,
42 v11: u8,
43 v12: u8,
44 v13: u8,
45 v14: u8,
46 v15: u8,
47 v16: u8,
48 v17: u8,
49 v18: u8,
50 v19: u8,
51 v20: u8,
52 v21: u8,
53 v22: u8,
54 v23: u8,
55 v24: u8,
56 v25: u8,
57 v26: u8,
58 v27: u8,
59 v28: u8,
60 v29: u8,
61 v30: u8,
62 v31: u8,
63 ) -> Self {
64 Self::from(_mm512_set_epi8(
65 v31 as i8, v30 as i8, v29 as i8, v28 as i8, v27 as i8, v26 as i8, v25 as i8, v24 as i8,
66 v23 as i8, v22 as i8, v21 as i8, v20 as i8, v19 as i8, v18 as i8, v17 as i8, v16 as i8,
67 v15 as i8, v14 as i8, v13 as i8, v12 as i8, v11 as i8, v10 as i8, v9 as i8, v8 as i8,
68 v7 as i8, v6 as i8, v5 as i8, v4 as i8, v3 as i8, v2 as i8, v1 as i8, v0 as i8,
69 v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8,
70 v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8,
71 v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8,
72 v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8,
73 ))
74 }
75
76 #[allow(clippy::too_many_arguments)]
77 #[allow(clippy::cast_possible_wrap)]
78 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
79 #[inline]
80 unsafe fn repeat_16(
81 v0: u8,
82 v1: u8,
83 v2: u8,
84 v3: u8,
85 v4: u8,
86 v5: u8,
87 v6: u8,
88 v7: u8,
89 v8: u8,
90 v9: u8,
91 v10: u8,
92 v11: u8,
93 v12: u8,
94 v13: u8,
95 v14: u8,
96 v15: u8,
97 ) -> Self {
98 Self::from(_mm512_set_epi8(
99 v15 as i8, v14 as i8, v13 as i8, v12 as i8, v11 as i8, v10 as i8, v9 as i8, v8 as i8,
100 v7 as i8, v6 as i8, v5 as i8, v4 as i8, v3 as i8, v2 as i8, v1 as i8, v0 as i8,
101 v15 as i8, v14 as i8, v13 as i8, v12 as i8, v11 as i8, v10 as i8, v9 as i8, v8 as i8,
102 v7 as i8, v6 as i8, v5 as i8, v4 as i8, v3 as i8, v2 as i8, v1 as i8, v0 as i8,
103 v15 as i8, v14 as i8, v13 as i8, v12 as i8, v11 as i8, v10 as i8, v9 as i8, v8 as i8,
104 v7 as i8, v6 as i8, v5 as i8, v4 as i8, v3 as i8, v2 as i8, v1 as i8, v0 as i8,
105 v15 as i8, v14 as i8, v13 as i8, v12 as i8, v11 as i8, v10 as i8, v9 as i8, v8 as i8,
106 v7 as i8, v6 as i8, v5 as i8, v4 as i8, v3 as i8, v2 as i8, v1 as i8, v0 as i8,
107 ))
108 }
109
110 #[allow(clippy::cast_ptr_alignment)]
111 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
112 #[inline]
113 unsafe fn load_from(ptr: *const u8) -> Self {
114 Self::from(_mm512_loadu_si512(ptr.cast::<__m512i>()))
115 }
116
117 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
118 #[inline]
119 unsafe fn load_from_partial(ptr: *const u8, len: usize) -> Self {
120 let res = _mm512_maskz_loadu_epi8(u64::MAX >> (64 - len), ptr.cast::<i8>());
121 Self::from(res)
122 }
123
124 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
125 #[inline]
126 unsafe fn lookup_16(self, tbl: Self) -> Self {
127 Self::from(_mm512_permutexvar_epi8(self.0, tbl.0))
131 }
132
133 #[allow(clippy::cast_possible_wrap)]
134 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
135 #[inline]
136 unsafe fn splat(val: u8) -> Self {
137 Self::from(_mm512_set1_epi8(val as i8))
138 }
139
140 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
141 #[inline]
142 unsafe fn splat0() -> Self {
143 Self::from(_mm512_setzero_si512())
144 }
145
146 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
147 #[inline]
148 unsafe fn or(self, b: Self) -> Self {
149 Self::from(_mm512_or_si512(self.0, b.0))
150 }
151
152 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
153 #[inline]
154 unsafe fn saturating_sub(self, b: Self) -> Self {
155 Self::from(_mm512_subs_epu8(self.0, b.0))
156 }
157
158 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
162 #[inline]
163 unsafe fn and3(self, b: Self, c: Self) -> Self {
164 Self::from(_mm512_ternarylogic_epi32(self.0, b.0, c.0, 0x80))
165 }
166
167 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
169 #[inline]
170 unsafe fn or_and(self, b: Self, c: Self) -> Self {
171 Self::from(_mm512_ternarylogic_epi32(self.0, b.0, c.0, 0xA8))
172 }
173
174 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
176 #[inline]
177 unsafe fn xor_or(self, b: Self, c: Self) -> Self {
178 Self::from(_mm512_ternarylogic_epi32(c.0, self.0, b.0, 0xF6))
179 }
180
181 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
185 #[inline]
186 unsafe fn shr4_dirty(self) -> Self {
187 Self::from(_mm512_srli_epi16(self.0, 4))
188 }
189
190 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
192 #[inline]
193 unsafe fn prev1(self, prev: Self) -> Self {
194 const SHIFT: i32 = 16 - 1;
195 return Self::from(_mm512_alignr_epi8(
196 self.0,
197 _mm512_permutex2var_epi64(prev.0, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), self.0),
198 SHIFT,
199 ));
200 }
201 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
203 #[inline]
204 unsafe fn prev2(self, prev: Self) -> Self {
205 const SHIFT: i32 = 16 - 2;
206 return Self::from(_mm512_alignr_epi8(
207 self.0,
208 _mm512_permutex2var_epi64(prev.0, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), self.0),
209 SHIFT,
210 ));
211 }
212
213 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
215 #[inline]
216 unsafe fn prev3(self, prev: Self) -> Self {
217 const SHIFT: i32 = 16 - 3;
218 return Self::from(_mm512_alignr_epi8(
219 self.0,
220 _mm512_permutex2var_epi64(prev.0, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), self.0),
221 SHIFT,
222 ));
223 }
224
225 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
226 #[inline]
227 unsafe fn any_bit_set(self) -> bool {
228 _mm512_test_epi8_mask(self.0, self.0) != 0
229 }
230
231 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
232 #[inline]
233 unsafe fn is_ascii(self) -> bool {
234 _mm512_movepi8_mask(self.0) == 0
235 }
236}
237
238impl From<__m512i> for SimdU8Value {
239 #[inline]
240 fn from(val: __m512i) -> Self {
241 Self(val)
242 }
243}
244
245impl Utf8CheckAlgorithm<SimdU8Value> {
247 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
248 #[inline]
249 unsafe fn check_special_cases(input: SimdU8Value, prev1: SimdU8Value) -> SimdU8Value {
250 let (byte_1_high_table, byte_1_low_table, byte_2_high_table) = Self::special_case_tables();
251
252 let byte_1_high = prev1.shr4_dirty().lookup_16(byte_1_high_table);
255 let byte_1_low = prev1.lookup_16(byte_1_low_table);
258 let byte_2_high = input.shr4_dirty().lookup_16(byte_2_high_table);
259
260 byte_1_high.and3(byte_1_low, byte_2_high)
261 }
262
263 #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
264 #[inline]
265 unsafe fn check_multibyte_lengths(
266 input: SimdU8Value,
267 prev: SimdU8Value,
268 special_cases: SimdU8Value,
269 error: SimdU8Value,
270 ) -> SimdU8Value {
271 let prev2 = input.prev2(prev);
272 let prev3 = input.prev3(prev);
273 let is_third_byte = prev2.saturating_sub(SimdU8Value::splat(0xe0 - 0x80));
274 let is_fourth_byte = prev3.saturating_sub(SimdU8Value::splat(0xf0 - 0x80));
275 let must23_80 = is_third_byte.or_and(is_fourth_byte, SimdU8Value::splat(0x80));
276 must23_80.xor_or(special_cases, error)
277 }
278}
279
280#[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
281#[inline]
282unsafe fn simd_prefetch(ptr: *const u8) {
283 _mm_prefetch(ptr.cast::<i8>(), _MM_HINT_T0);
284}
285
286const PREFETCH: bool = true;
287#[cfg(feature = "public_imp")]
288use crate::implementation::helpers::TempSimdChunkA64 as TempSimdChunk;
289simd_input_512_bit!(#[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]);
290algorithm_simd!(#[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]);