1#[cfg(target_arch = "x86")]
4use core::arch::x86::{
5 __m256i, _mm256_alignr_epi8, _mm256_and_si256, _mm256_loadu_si256, _mm256_movemask_epi8,
6 _mm256_or_si256, _mm256_permute2x128_si256, _mm256_set1_epi8, _mm256_setr_epi8,
7 _mm256_setzero_si256, _mm256_shuffle_epi8, _mm256_srli_epi16, _mm256_subs_epu8,
8 _mm256_testz_si256, _mm256_xor_si256, _mm_prefetch, _MM_HINT_T0,
9};
10#[cfg(target_arch = "x86_64")]
11use core::arch::x86_64::{
12 __m256i, _mm256_alignr_epi8, _mm256_and_si256, _mm256_loadu_si256, _mm256_movemask_epi8,
13 _mm256_or_si256, _mm256_permute2x128_si256, _mm256_set1_epi8, _mm256_setr_epi8,
14 _mm256_setzero_si256, _mm256_shuffle_epi8, _mm256_srli_epi16, _mm256_subs_epu8,
15 _mm256_testz_si256, _mm256_xor_si256, _mm_prefetch, _MM_HINT_T0,
16};
17
18use crate::implementation::helpers::Utf8CheckAlgorithm;
19
20type SimdU8Value = crate::implementation::helpers::SimdU8Value<__m256i>;
23
24impl SimdU8Value {
25 #[allow(clippy::cast_possible_wrap)]
26 #[allow(clippy::too_many_arguments)]
27 #[target_feature(enable = "avx2")]
28 #[inline]
29 unsafe fn from_32_cut_off_leading(
30 v0: u8,
31 v1: u8,
32 v2: u8,
33 v3: u8,
34 v4: u8,
35 v5: u8,
36 v6: u8,
37 v7: u8,
38 v8: u8,
39 v9: u8,
40 v10: u8,
41 v11: u8,
42 v12: u8,
43 v13: u8,
44 v14: u8,
45 v15: u8,
46 v16: u8,
47 v17: u8,
48 v18: u8,
49 v19: u8,
50 v20: u8,
51 v21: u8,
52 v22: u8,
53 v23: u8,
54 v24: u8,
55 v25: u8,
56 v26: u8,
57 v27: u8,
58 v28: u8,
59 v29: u8,
60 v30: u8,
61 v31: u8,
62 ) -> Self {
63 Self::from(_mm256_setr_epi8(
64 v0 as i8, v1 as i8, v2 as i8, v3 as i8, v4 as i8, v5 as i8, v6 as i8, v7 as i8,
65 v8 as i8, v9 as i8, v10 as i8, v11 as i8, v12 as i8, v13 as i8, v14 as i8, v15 as i8,
66 v16 as i8, v17 as i8, v18 as i8, v19 as i8, v20 as i8, v21 as i8, v22 as i8, v23 as i8,
67 v24 as i8, v25 as i8, v26 as i8, v27 as i8, v28 as i8, v29 as i8, v30 as i8, v31 as i8,
68 ))
69 }
70
71 #[allow(clippy::too_many_arguments)]
72 #[target_feature(enable = "avx2")]
73 #[inline]
74 unsafe fn repeat_16(
75 v0: u8,
76 v1: u8,
77 v2: u8,
78 v3: u8,
79 v4: u8,
80 v5: u8,
81 v6: u8,
82 v7: u8,
83 v8: u8,
84 v9: u8,
85 v10: u8,
86 v11: u8,
87 v12: u8,
88 v13: u8,
89 v14: u8,
90 v15: u8,
91 ) -> Self {
92 Self::from_32_cut_off_leading(
93 v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v0, v1, v2, v3,
94 v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15,
95 )
96 }
97
98 #[allow(clippy::cast_ptr_alignment)]
99 #[target_feature(enable = "avx2")]
100 #[inline]
101 unsafe fn load_from(ptr: *const u8) -> Self {
102 Self::from(_mm256_loadu_si256(ptr.cast::<__m256i>()))
103 }
104
105 #[target_feature(enable = "avx2")]
106 #[inline]
107 unsafe fn lookup_16(self, tbl: Self) -> Self {
108 Self::from(_mm256_shuffle_epi8(tbl.0, self.0))
109 }
110
111 #[allow(clippy::cast_possible_wrap)]
112 #[target_feature(enable = "avx2")]
113 #[inline]
114 unsafe fn splat(val: u8) -> Self {
115 Self::from(_mm256_set1_epi8(val as i8))
116 }
117
118 #[target_feature(enable = "avx2")]
119 #[inline]
120 unsafe fn splat0() -> Self {
121 Self::from(_mm256_setzero_si256())
122 }
123
124 #[target_feature(enable = "avx2")]
125 #[inline]
126 unsafe fn or(self, b: Self) -> Self {
127 Self::from(_mm256_or_si256(self.0, b.0))
128 }
129
130 #[target_feature(enable = "avx2")]
131 #[inline]
132 unsafe fn and(self, b: Self) -> Self {
133 Self::from(_mm256_and_si256(self.0, b.0))
134 }
135
136 #[target_feature(enable = "avx2")]
137 #[inline]
138 unsafe fn xor(self, b: Self) -> Self {
139 Self::from(_mm256_xor_si256(self.0, b.0))
140 }
141
142 #[target_feature(enable = "avx2")]
143 #[inline]
144 unsafe fn saturating_sub(self, b: Self) -> Self {
145 Self::from(_mm256_subs_epu8(self.0, b.0))
146 }
147
148 #[target_feature(enable = "avx2")]
150 #[inline]
151 unsafe fn shr4(self) -> Self {
152 Self::from(_mm256_srli_epi16(self.0, 4)).and(Self::splat(0xFF >> 4))
153 }
154
155 #[target_feature(enable = "avx2")]
157 #[inline]
158 unsafe fn prev1(self, prev: Self) -> Self {
159 Self::from(_mm256_alignr_epi8(
160 self.0,
161 _mm256_permute2x128_si256(prev.0, self.0, 0x21),
162 16 - 1,
163 ))
164 }
165
166 #[target_feature(enable = "avx2")]
168 #[inline]
169 unsafe fn prev2(self, prev: Self) -> Self {
170 Self::from(_mm256_alignr_epi8(
171 self.0,
172 _mm256_permute2x128_si256(prev.0, self.0, 0x21),
173 16 - 2,
174 ))
175 }
176
177 #[target_feature(enable = "avx2")]
179 #[inline]
180 unsafe fn prev3(self, prev: Self) -> Self {
181 Self::from(_mm256_alignr_epi8(
182 self.0,
183 _mm256_permute2x128_si256(prev.0, self.0, 0x21),
184 16 - 3,
185 ))
186 }
187
188 #[target_feature(enable = "avx2")]
189 #[inline]
190 unsafe fn any_bit_set(self) -> bool {
191 _mm256_testz_si256(self.0, self.0) != 1
192 }
193
194 #[target_feature(enable = "avx2")]
195 #[inline]
196 unsafe fn is_ascii(self) -> bool {
197 _mm256_movemask_epi8(self.0) == 0
198 }
199}
200
201impl From<__m256i> for SimdU8Value {
202 #[inline]
203 fn from(val: __m256i) -> Self {
204 Self(val)
205 }
206}
207
208#[target_feature(enable = "avx2")]
209#[inline]
210unsafe fn simd_prefetch(ptr: *const u8) {
211 _mm_prefetch(ptr.cast::<i8>(), _MM_HINT_T0);
212}
213
214const PREFETCH: bool = true;
215use crate::implementation::helpers::TempSimdChunkA32 as TempSimdChunk;
216simd_input_256_bit!(#[target_feature(enable = "avx2")]);
217algorithm_simd!(#[target_feature(enable = "avx2")]);
218algorithm_simd_default_special_case_fns!(#[target_feature(enable = "avx2")]);