Skip to main content

simdutf8/implementation/x86/
avx512.rs

1//! Contains the x86-64 AVX512 UTF-8 validation implementation.
2
3#[cfg(target_arch = "x86")]
4use core::arch::x86::{
5    __m512i, _mm512_alignr_epi8, _mm512_loadu_si512, _mm512_maskz_loadu_epi8, _mm512_movepi8_mask,
6    _mm512_or_si512, _mm512_permutex2var_epi64, _mm512_permutexvar_epi8, _mm512_set1_epi8,
7    _mm512_set_epi64, _mm512_set_epi8, _mm512_setzero_si512, _mm512_srli_epi16, _mm512_subs_epu8,
8    _mm512_ternarylogic_epi32, _mm512_test_epi8_mask, _mm_prefetch, _MM_HINT_T0,
9};
10
11#[cfg(target_arch = "x86_64")]
12use core::arch::x86_64::{
13    __m512i, _mm512_alignr_epi8, _mm512_loadu_si512, _mm512_maskz_loadu_epi8, _mm512_movepi8_mask,
14    _mm512_or_si512, _mm512_permutex2var_epi64, _mm512_permutexvar_epi8, _mm512_set1_epi8,
15    _mm512_set_epi64, _mm512_set_epi8, _mm512_setzero_si512, _mm512_srli_epi16, _mm512_subs_epu8,
16    _mm512_ternarylogic_epi32, _mm512_test_epi8_mask, _mm_prefetch, _MM_HINT_T0,
17};
18
19use crate::implementation::helpers::Utf8CheckAlgorithm;
20
21// AVX 2 SIMD primitives
22
23type SimdU8Value = crate::implementation::helpers::SimdU8Value<__m512i>;
24
25impl SimdU8Value {
26    #[allow(clippy::cast_possible_wrap)]
27    #[allow(clippy::too_many_arguments)]
28    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
29    #[inline]
30    unsafe fn from_32_cut_off_leading(
31        v0: u8,
32        v1: u8,
33        v2: u8,
34        v3: u8,
35        v4: u8,
36        v5: u8,
37        v6: u8,
38        v7: u8,
39        v8: u8,
40        v9: u8,
41        v10: u8,
42        v11: u8,
43        v12: u8,
44        v13: u8,
45        v14: u8,
46        v15: u8,
47        v16: u8,
48        v17: u8,
49        v18: u8,
50        v19: u8,
51        v20: u8,
52        v21: u8,
53        v22: u8,
54        v23: u8,
55        v24: u8,
56        v25: u8,
57        v26: u8,
58        v27: u8,
59        v28: u8,
60        v29: u8,
61        v30: u8,
62        v31: u8,
63    ) -> Self {
64        Self::from(_mm512_set_epi8(
65            v31 as i8, v30 as i8, v29 as i8, v28 as i8, v27 as i8, v26 as i8, v25 as i8, v24 as i8,
66            v23 as i8, v22 as i8, v21 as i8, v20 as i8, v19 as i8, v18 as i8, v17 as i8, v16 as i8,
67            v15 as i8, v14 as i8, v13 as i8, v12 as i8, v11 as i8, v10 as i8, v9 as i8, v8 as i8,
68            v7 as i8, v6 as i8, v5 as i8, v4 as i8, v3 as i8, v2 as i8, v1 as i8, v0 as i8,
69            v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8,
70            v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8,
71            v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8,
72            v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8, v0 as i8,
73        ))
74    }
75
76    #[allow(clippy::too_many_arguments)]
77    #[allow(clippy::cast_possible_wrap)]
78    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
79    #[inline]
80    unsafe fn repeat_16(
81        v0: u8,
82        v1: u8,
83        v2: u8,
84        v3: u8,
85        v4: u8,
86        v5: u8,
87        v6: u8,
88        v7: u8,
89        v8: u8,
90        v9: u8,
91        v10: u8,
92        v11: u8,
93        v12: u8,
94        v13: u8,
95        v14: u8,
96        v15: u8,
97    ) -> Self {
98        Self::from(_mm512_set_epi8(
99            v15 as i8, v14 as i8, v13 as i8, v12 as i8, v11 as i8, v10 as i8, v9 as i8, v8 as i8,
100            v7 as i8, v6 as i8, v5 as i8, v4 as i8, v3 as i8, v2 as i8, v1 as i8, v0 as i8,
101            v15 as i8, v14 as i8, v13 as i8, v12 as i8, v11 as i8, v10 as i8, v9 as i8, v8 as i8,
102            v7 as i8, v6 as i8, v5 as i8, v4 as i8, v3 as i8, v2 as i8, v1 as i8, v0 as i8,
103            v15 as i8, v14 as i8, v13 as i8, v12 as i8, v11 as i8, v10 as i8, v9 as i8, v8 as i8,
104            v7 as i8, v6 as i8, v5 as i8, v4 as i8, v3 as i8, v2 as i8, v1 as i8, v0 as i8,
105            v15 as i8, v14 as i8, v13 as i8, v12 as i8, v11 as i8, v10 as i8, v9 as i8, v8 as i8,
106            v7 as i8, v6 as i8, v5 as i8, v4 as i8, v3 as i8, v2 as i8, v1 as i8, v0 as i8,
107        ))
108    }
109
110    #[allow(clippy::cast_ptr_alignment)]
111    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
112    #[inline]
113    unsafe fn load_from(ptr: *const u8) -> Self {
114        Self::from(_mm512_loadu_si512(ptr.cast::<__m512i>()))
115    }
116
117    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
118    #[inline]
119    unsafe fn load_from_partial(ptr: *const u8, len: usize) -> Self {
120        let res = _mm512_maskz_loadu_epi8(u64::MAX >> (64 - len), ptr.cast::<i8>());
121        Self::from(res)
122    }
123
124    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
125    #[inline]
126    unsafe fn lookup_16(self, tbl: Self) -> Self {
127        // VPERMB (avx512vbmi) differs from other dynamic swizzle instructions in that it
128        // completely ignores the high bits of the index. Only the low 6 bits of each
129        // byte are used to select a byte from the 64-byte table.
130        Self::from(_mm512_permutexvar_epi8(self.0, tbl.0))
131    }
132
133    #[allow(clippy::cast_possible_wrap)]
134    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
135    #[inline]
136    unsafe fn splat(val: u8) -> Self {
137        Self::from(_mm512_set1_epi8(val as i8))
138    }
139
140    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
141    #[inline]
142    unsafe fn splat0() -> Self {
143        Self::from(_mm512_setzero_si512())
144    }
145
146    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
147    #[inline]
148    unsafe fn or(self, b: Self) -> Self {
149        Self::from(_mm512_or_si512(self.0, b.0))
150    }
151
152    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
153    #[inline]
154    unsafe fn saturating_sub(self, b: Self) -> Self {
155        Self::from(_mm512_subs_epu8(self.0, b.0))
156    }
157
158    // For ternary ops reference see https://www.felixcloutier.com/x86/vpternlogd:vpternlogq
159
160    /// `self & b & c` fused into a single `vpternlogd`.
161    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
162    #[inline]
163    unsafe fn and3(self, b: Self, c: Self) -> Self {
164        Self::from(_mm512_ternarylogic_epi32(self.0, b.0, c.0, 0x80))
165    }
166
167    /// `(self | b) & c` fused into a single `vpternlogd`.
168    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
169    #[inline]
170    unsafe fn or_and(self, b: Self, c: Self) -> Self {
171        Self::from(_mm512_ternarylogic_epi32(self.0, b.0, c.0, 0xA8))
172    }
173
174    /// `(self ^ b) | c` fused into a single `vpternlogd`.
175    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
176    #[inline]
177    unsafe fn xor_or(self, b: Self, c: Self) -> Self {
178        Self::from(_mm512_ternarylogic_epi32(c.0, self.0, b.0, 0xF6))
179    }
180
181    /// CAVE: Uses 16-bit-lane shifts so the high nibble of every
182    /// other byte is polluted with the low nibble of the neighbouring byte.
183    /// This is fine for our use case since we only care about the low nibble.
184    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
185    #[inline]
186    unsafe fn shr4_dirty(self) -> Self {
187        Self::from(_mm512_srli_epi16(self.0, 4))
188    }
189
190    // ugly but prev<N> requires const generics
191    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
192    #[inline]
193    unsafe fn prev1(self, prev: Self) -> Self {
194        const SHIFT: i32 = 16 - 1;
195        return Self::from(_mm512_alignr_epi8(
196            self.0,
197            _mm512_permutex2var_epi64(prev.0, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), self.0),
198            SHIFT,
199        ));
200    }
201    // ugly but prev<N> requires const generics
202    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
203    #[inline]
204    unsafe fn prev2(self, prev: Self) -> Self {
205        const SHIFT: i32 = 16 - 2;
206        return Self::from(_mm512_alignr_epi8(
207            self.0,
208            _mm512_permutex2var_epi64(prev.0, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), self.0),
209            SHIFT,
210        ));
211    }
212
213    // ugly but prev<N> requires const generics
214    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
215    #[inline]
216    unsafe fn prev3(self, prev: Self) -> Self {
217        const SHIFT: i32 = 16 - 3;
218        return Self::from(_mm512_alignr_epi8(
219            self.0,
220            _mm512_permutex2var_epi64(prev.0, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), self.0),
221            SHIFT,
222        ));
223    }
224
225    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
226    #[inline]
227    unsafe fn any_bit_set(self) -> bool {
228        _mm512_test_epi8_mask(self.0, self.0) != 0
229    }
230
231    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
232    #[inline]
233    unsafe fn is_ascii(self) -> bool {
234        _mm512_movepi8_mask(self.0) == 0
235    }
236}
237
238impl From<__m512i> for SimdU8Value {
239    #[inline]
240    fn from(val: __m512i) -> Self {
241        Self(val)
242    }
243}
244
245/// AVX-512 specializations of `check_special_cases` and `check_multibyte_lengths`.
246impl Utf8CheckAlgorithm<SimdU8Value> {
247    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
248    #[inline]
249    unsafe fn check_special_cases(input: SimdU8Value, prev1: SimdU8Value) -> SimdU8Value {
250        let (byte_1_high_table, byte_1_low_table, byte_2_high_table) = Self::special_case_tables();
251
252        // `lookup_16` (VPERMB) on tables where the 128-bit lanes are all identical
253        // ignores the high nibble, so the pollution by `shr4_dirty` does not matter.
254        let byte_1_high = prev1.shr4_dirty().lookup_16(byte_1_high_table);
255        // `lookup_16` (VPERMB) on tables where the 128-bit lanes are all identical
256        // ignores the high nibble, so no masking needed.
257        let byte_1_low = prev1.lookup_16(byte_1_low_table);
258        let byte_2_high = input.shr4_dirty().lookup_16(byte_2_high_table);
259
260        byte_1_high.and3(byte_1_low, byte_2_high)
261    }
262
263    #[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
264    #[inline]
265    unsafe fn check_multibyte_lengths(
266        input: SimdU8Value,
267        prev: SimdU8Value,
268        special_cases: SimdU8Value,
269        error: SimdU8Value,
270    ) -> SimdU8Value {
271        let prev2 = input.prev2(prev);
272        let prev3 = input.prev3(prev);
273        let is_third_byte = prev2.saturating_sub(SimdU8Value::splat(0xe0 - 0x80));
274        let is_fourth_byte = prev3.saturating_sub(SimdU8Value::splat(0xf0 - 0x80));
275        let must23_80 = is_third_byte.or_and(is_fourth_byte, SimdU8Value::splat(0x80));
276        must23_80.xor_or(special_cases, error)
277    }
278}
279
280#[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]
281#[inline]
282unsafe fn simd_prefetch(ptr: *const u8) {
283    _mm_prefetch(ptr.cast::<i8>(), _MM_HINT_T0);
284}
285
286const PREFETCH: bool = true;
287#[cfg(feature = "public_imp")]
288use crate::implementation::helpers::TempSimdChunkA64 as TempSimdChunk;
289simd_input_512_bit!(#[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]);
290algorithm_simd!(#[target_feature(enable = "avx512f,avx512bw,avx512vbmi")]);