1use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal};
7use crate::{
8 f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4,
9 i32x8, i32x16, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, mask16x32, mask32x4,
10 mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, u16x8, u16x16, u16x32,
11 u32x4, u32x8, u32x16,
12};
13#[cfg(target_arch = "x86")]
14use core::arch::x86::*;
15#[cfg(target_arch = "x86_64")]
16use core::arch::x86_64::*;
17#[doc = "A token for SSE4.2 intrinsics on `x86` and `x86_64`, representing the x86-64-v2 level."]
18#[derive(Clone, Copy, Debug)]
19pub struct Sse4_2 {
20 _private: (),
21}
22impl Sse4_2 {
23 #[doc = r" Create a SIMD token."]
24 #[doc = r""]
25 #[doc = r" # Safety"]
26 #[doc = r""]
27 #[doc = r" The `sse4.2`, `cmpxchg16b`, and `popcnt` CPU features must"]
28 #[doc = r" be available."]
29 #[inline]
30 pub const unsafe fn new_unchecked() -> Self {
31 Sse4_2 { _private: () }
32 }
33}
34impl Seal for Sse4_2 {}
35impl ArchTypes for Sse4_2 {
36 type f32x4 = crate::support::Aligned128<__m128>;
37 type i8x16 = crate::support::Aligned128<__m128i>;
38 type u8x16 = crate::support::Aligned128<__m128i>;
39 type mask8x16 = crate::support::Aligned128<__m128i>;
40 type i16x8 = crate::support::Aligned128<__m128i>;
41 type u16x8 = crate::support::Aligned128<__m128i>;
42 type mask16x8 = crate::support::Aligned128<__m128i>;
43 type i32x4 = crate::support::Aligned128<__m128i>;
44 type u32x4 = crate::support::Aligned128<__m128i>;
45 type mask32x4 = crate::support::Aligned128<__m128i>;
46 type f64x2 = crate::support::Aligned128<__m128d>;
47 type mask64x2 = crate::support::Aligned128<__m128i>;
48 type f32x8 = crate::support::Aligned256<[__m128; 2usize]>;
49 type i8x32 = crate::support::Aligned256<[__m128i; 2usize]>;
50 type u8x32 = crate::support::Aligned256<[__m128i; 2usize]>;
51 type mask8x32 = crate::support::Aligned256<[__m128i; 2usize]>;
52 type i16x16 = crate::support::Aligned256<[__m128i; 2usize]>;
53 type u16x16 = crate::support::Aligned256<[__m128i; 2usize]>;
54 type mask16x16 = crate::support::Aligned256<[__m128i; 2usize]>;
55 type i32x8 = crate::support::Aligned256<[__m128i; 2usize]>;
56 type u32x8 = crate::support::Aligned256<[__m128i; 2usize]>;
57 type mask32x8 = crate::support::Aligned256<[__m128i; 2usize]>;
58 type f64x4 = crate::support::Aligned256<[__m128d; 2usize]>;
59 type mask64x4 = crate::support::Aligned256<[__m128i; 2usize]>;
60 type f32x16 = crate::support::Aligned512<[__m128; 4usize]>;
61 type i8x64 = crate::support::Aligned512<[__m128i; 4usize]>;
62 type u8x64 = crate::support::Aligned512<[__m128i; 4usize]>;
63 type mask8x64 = crate::support::Aligned512<[__m128i; 4usize]>;
64 type i16x32 = crate::support::Aligned512<[__m128i; 4usize]>;
65 type u16x32 = crate::support::Aligned512<[__m128i; 4usize]>;
66 type mask16x32 = crate::support::Aligned512<[__m128i; 4usize]>;
67 type i32x16 = crate::support::Aligned512<[__m128i; 4usize]>;
68 type u32x16 = crate::support::Aligned512<[__m128i; 4usize]>;
69 type mask32x16 = crate::support::Aligned512<[__m128i; 4usize]>;
70 type f64x8 = crate::support::Aligned512<[__m128d; 4usize]>;
71 type mask64x8 = crate::support::Aligned512<[__m128i; 4usize]>;
72}
73impl Simd for Sse4_2 {
74 type f32s = f32x4<Self>;
75 type f64s = f64x2<Self>;
76 type u8s = u8x16<Self>;
77 type i8s = i8x16<Self>;
78 type u16s = u16x8<Self>;
79 type i16s = i16x8<Self>;
80 type u32s = u32x4<Self>;
81 type i32s = i32x4<Self>;
82 type mask8s = mask8x16<Self>;
83 type mask16s = mask16x8<Self>;
84 type mask32s = mask32x4<Self>;
85 type mask64s = mask64x2<Self>;
86 #[inline(always)]
87 fn level(self) -> Level {
88 Level::Sse4_2(self)
89 }
90 #[inline]
91 fn vectorize<F: FnOnce() -> R, R>(self, f: F) -> R {
92 #[target_feature(enable = "sse4.2,cmpxchg16b,popcnt")]
93 fn vectorize_sse4_2<F: FnOnce() -> R, R>(f: F) -> R {
94 f()
95 }
96 unsafe { vectorize_sse4_2(f) }
97 }
98 #[inline(always)]
99 fn splat_f32x4(self, val: f32) -> f32x4<Self> {
100 crate::kernel!(
101 #[inline(always)]
102 fn kernel(token: Sse4_2, val: f32) -> f32x4<Sse4_2> {
103 _mm_set1_ps(val).simd_into(token)
104 }
105 );
106 kernel(self, val)
107 }
108 #[inline(always)]
109 fn load_array_f32x4(self, val: [f32; 4usize]) -> f32x4<Self> {
110 f32x4 {
111 val: crate::transmute::checked_transmute_copy(&val),
112 simd: self,
113 }
114 }
115 #[inline(always)]
116 fn load_array_ref_f32x4(self, val: &[f32; 4usize]) -> f32x4<Self> {
117 f32x4 {
118 val: crate::transmute::checked_transmute_copy(val),
119 simd: self,
120 }
121 }
122 #[inline(always)]
123 fn as_array_f32x4(self, a: f32x4<Self>) -> [f32; 4usize] {
124 crate::transmute::checked_transmute_copy::<__m128, [f32; 4usize]>(&a.val.0)
125 }
126 #[inline(always)]
127 fn as_array_ref_f32x4(self, a: &f32x4<Self>) -> &[f32; 4usize] {
128 crate::transmute::checked_cast_ref::<__m128, [f32; 4usize]>(&a.val.0)
129 }
130 #[inline(always)]
131 fn as_array_mut_f32x4(self, a: &mut f32x4<Self>) -> &mut [f32; 4usize] {
132 crate::transmute::checked_cast_mut::<__m128, [f32; 4usize]>(&mut a.val.0)
133 }
134 #[inline(always)]
135 fn store_array_f32x4(self, a: f32x4<Self>, dest: &mut [f32; 4usize]) -> () {
136 crate::transmute::checked_transmute_store(a.val.0, dest);
137 }
138 #[inline(always)]
139 fn cvt_from_bytes_f32x4(self, a: u8x16<Self>) -> f32x4<Self> {
140 f32x4 {
141 val: crate::transmute::checked_transmute_copy(&a.val),
142 simd: self,
143 }
144 }
145 #[inline(always)]
146 fn cvt_to_bytes_f32x4(self, a: f32x4<Self>) -> u8x16<Self> {
147 u8x16 {
148 val: crate::transmute::checked_transmute_copy(&a.val),
149 simd: self,
150 }
151 }
152 #[inline(always)]
153 fn slide_f32x4<const SHIFT: usize>(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
154 if SHIFT >= 4usize {
155 return b;
156 }
157 let result = dyn_alignr_128(
158 self,
159 self.cvt_to_bytes_f32x4(b).val.0,
160 self.cvt_to_bytes_f32x4(a).val.0,
161 SHIFT * 4usize,
162 );
163 self.cvt_from_bytes_f32x4(u8x16 {
164 val: crate::support::Aligned128(result),
165 simd: self,
166 })
167 }
168 #[inline(always)]
169 fn slide_within_blocks_f32x4<const SHIFT: usize>(
170 self,
171 a: f32x4<Self>,
172 b: f32x4<Self>,
173 ) -> f32x4<Self> {
174 self.slide_f32x4::<SHIFT>(a, b)
175 }
176 #[inline(always)]
177 fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4<Self>, indices: u8x16<Self>) -> f32x4<Self> {
178 crate::kernel!(
179 #[inline(always)]
180 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, indices: u8x16<Sse4_2>) -> f32x4<Sse4_2> {
181 let result = _mm_shuffle_epi8(token.cvt_to_bytes_f32x4(a).val.0, indices.into());
182 token.cvt_from_bytes_f32x4(u8x16 {
183 val: crate::support::Aligned128(result),
184 simd: token,
185 })
186 }
187 );
188 kernel(self, a, indices)
189 }
190 #[inline(always)]
191 fn abs_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
192 crate::kernel!(
193 #[inline(always)]
194 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
195 _mm_andnot_ps(_mm_set1_ps(-0.0), a.into()).simd_into(token)
196 }
197 );
198 kernel(self, a)
199 }
200 #[inline(always)]
201 fn neg_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
202 crate::kernel!(
203 #[inline(always)]
204 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
205 _mm_xor_ps(a.into(), _mm_set1_ps(-0.0)).simd_into(token)
206 }
207 );
208 kernel(self, a)
209 }
210 #[inline(always)]
211 fn sqrt_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
212 crate::kernel!(
213 #[inline(always)]
214 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
215 _mm_sqrt_ps(a.into()).simd_into(token)
216 }
217 );
218 kernel(self, a)
219 }
220 #[inline(always)]
221 fn approximate_recip_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
222 crate::kernel!(
223 #[inline(always)]
224 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
225 _mm_rcp_ps(a.into()).simd_into(token)
226 }
227 );
228 kernel(self, a)
229 }
230 #[inline(always)]
231 fn add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
232 crate::kernel!(
233 #[inline(always)]
234 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
235 _mm_add_ps(a.into(), b.into()).simd_into(token)
236 }
237 );
238 kernel(self, a, b)
239 }
240 #[inline(always)]
241 fn sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
242 crate::kernel!(
243 #[inline(always)]
244 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
245 _mm_sub_ps(a.into(), b.into()).simd_into(token)
246 }
247 );
248 kernel(self, a, b)
249 }
250 #[inline(always)]
251 fn mul_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
252 crate::kernel!(
253 #[inline(always)]
254 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
255 _mm_mul_ps(a.into(), b.into()).simd_into(token)
256 }
257 );
258 kernel(self, a, b)
259 }
260 #[inline(always)]
261 fn div_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
262 crate::kernel!(
263 #[inline(always)]
264 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
265 _mm_div_ps(a.into(), b.into()).simd_into(token)
266 }
267 );
268 kernel(self, a, b)
269 }
270 #[inline(always)]
271 fn copysign_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
272 crate::kernel!(
273 #[inline(always)]
274 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
275 let mask = _mm_set1_ps(-0.0);
276 _mm_or_ps(_mm_and_ps(mask, b.into()), _mm_andnot_ps(mask, a.into()))
277 .simd_into(token)
278 }
279 );
280 kernel(self, a, b)
281 }
282 #[inline(always)]
283 fn simd_eq_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
284 crate::kernel!(
285 #[inline(always)]
286 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> mask32x4<Sse4_2> {
287 _mm_castps_si128(_mm_cmpeq_ps(a.into(), b.into())).simd_into(token)
288 }
289 );
290 kernel(self, a, b)
291 }
292 #[inline(always)]
293 fn simd_lt_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
294 crate::kernel!(
295 #[inline(always)]
296 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> mask32x4<Sse4_2> {
297 _mm_castps_si128(_mm_cmplt_ps(a.into(), b.into())).simd_into(token)
298 }
299 );
300 kernel(self, a, b)
301 }
302 #[inline(always)]
303 fn simd_le_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
304 crate::kernel!(
305 #[inline(always)]
306 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> mask32x4<Sse4_2> {
307 _mm_castps_si128(_mm_cmple_ps(a.into(), b.into())).simd_into(token)
308 }
309 );
310 kernel(self, a, b)
311 }
312 #[inline(always)]
313 fn simd_ge_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
314 crate::kernel!(
315 #[inline(always)]
316 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> mask32x4<Sse4_2> {
317 _mm_castps_si128(_mm_cmpge_ps(a.into(), b.into())).simd_into(token)
318 }
319 );
320 kernel(self, a, b)
321 }
322 #[inline(always)]
323 fn simd_gt_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
324 crate::kernel!(
325 #[inline(always)]
326 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> mask32x4<Sse4_2> {
327 _mm_castps_si128(_mm_cmpgt_ps(a.into(), b.into())).simd_into(token)
328 }
329 );
330 kernel(self, a, b)
331 }
332 #[inline(always)]
333 fn zip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
334 crate::kernel!(
335 #[inline(always)]
336 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
337 _mm_unpacklo_ps(a.into(), b.into()).simd_into(token)
338 }
339 );
340 kernel(self, a, b)
341 }
342 #[inline(always)]
343 fn zip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
344 crate::kernel!(
345 #[inline(always)]
346 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
347 _mm_unpackhi_ps(a.into(), b.into()).simd_into(token)
348 }
349 );
350 kernel(self, a, b)
351 }
352 #[inline(always)]
353 fn unzip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
354 crate::kernel!(
355 #[inline(always)]
356 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
357 _mm_shuffle_ps::<0b10_00_10_00>(a.into(), b.into()).simd_into(token)
358 }
359 );
360 kernel(self, a, b)
361 }
362 #[inline(always)]
363 fn unzip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
364 crate::kernel!(
365 #[inline(always)]
366 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
367 _mm_shuffle_ps::<0b11_01_11_01>(a.into(), b.into()).simd_into(token)
368 }
369 );
370 kernel(self, a, b)
371 }
372 #[inline(always)]
373 fn interleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
374 (self.zip_low_f32x4(a, b), self.zip_high_f32x4(a, b))
375 }
376 #[inline(always)]
377 fn deinterleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
378 (self.unzip_low_f32x4(a, b), self.unzip_high_f32x4(a, b))
379 }
380 #[inline(always)]
381 fn max_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
382 crate::kernel!(
383 #[inline(always)]
384 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
385 _mm_max_ps(a.into(), b.into()).simd_into(token)
386 }
387 );
388 kernel(self, a, b)
389 }
390 #[inline(always)]
391 fn min_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
392 crate::kernel!(
393 #[inline(always)]
394 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
395 _mm_min_ps(a.into(), b.into()).simd_into(token)
396 }
397 );
398 kernel(self, a, b)
399 }
400 #[inline(always)]
401 fn max_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
402 crate::kernel!(
403 #[inline(always)]
404 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
405 let intermediate = _mm_max_ps(a.into(), b.into());
406 let b_is_nan = _mm_cmpunord_ps(b.into(), b.into());
407 _mm_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token)
408 }
409 );
410 kernel(self, a, b)
411 }
412 #[inline(always)]
413 fn min_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
414 crate::kernel!(
415 #[inline(always)]
416 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
417 let intermediate = _mm_min_ps(a.into(), b.into());
418 let b_is_nan = _mm_cmpunord_ps(b.into(), b.into());
419 _mm_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token)
420 }
421 );
422 kernel(self, a, b)
423 }
424 #[inline(always)]
425 fn mul_add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
426 a * b + c
427 }
428 #[inline(always)]
429 fn mul_sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
430 a * b - c
431 }
432 #[inline(always)]
433 fn floor_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
434 crate::kernel!(
435 #[inline(always)]
436 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
437 _mm_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
438 .simd_into(token)
439 }
440 );
441 kernel(self, a)
442 }
443 #[inline(always)]
444 fn ceil_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
445 crate::kernel!(
446 #[inline(always)]
447 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
448 _mm_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
449 .simd_into(token)
450 }
451 );
452 kernel(self, a)
453 }
454 #[inline(always)]
455 fn round_ties_even_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
456 crate::kernel!(
457 #[inline(always)]
458 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
459 _mm_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
460 .simd_into(token)
461 }
462 );
463 kernel(self, a)
464 }
465 #[inline(always)]
466 fn fract_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
467 a - self.trunc_f32x4(a)
468 }
469 #[inline(always)]
470 fn trunc_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
471 crate::kernel!(
472 #[inline(always)]
473 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
474 _mm_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
475 .simd_into(token)
476 }
477 );
478 kernel(self, a)
479 }
480 #[inline(always)]
481 fn select_f32x4(self, a: mask32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
482 crate::kernel!(
483 #[inline(always)]
484 fn kernel(
485 token: Sse4_2,
486 a: mask32x4<Sse4_2>,
487 b: f32x4<Sse4_2>,
488 c: f32x4<Sse4_2>,
489 ) -> f32x4<Sse4_2> {
490 _mm_blendv_ps(c.into(), b.into(), _mm_castsi128_ps(a.into())).simd_into(token)
491 }
492 );
493 kernel(self, a, b, c)
494 }
495 #[inline(always)]
496 fn combine_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x8<Self> {
497 f32x8 {
498 val: crate::support::Aligned256([a.val.0, b.val.0]),
499 simd: self,
500 }
501 }
502 #[inline(always)]
503 fn reinterpret_f64_f32x4(self, a: f32x4<Self>) -> f64x2<Self> {
504 crate::kernel!(
505 #[inline(always)]
506 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f64x2<Sse4_2> {
507 _mm_castps_pd(a.into()).simd_into(token)
508 }
509 );
510 kernel(self, a)
511 }
512 #[inline(always)]
513 fn reinterpret_i32_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
514 crate::kernel!(
515 #[inline(always)]
516 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> i32x4<Sse4_2> {
517 _mm_castps_si128(a.into()).simd_into(token)
518 }
519 );
520 kernel(self, a)
521 }
522 #[inline(always)]
523 fn reinterpret_u8_f32x4(self, a: f32x4<Self>) -> u8x16<Self> {
524 crate::kernel!(
525 #[inline(always)]
526 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> u8x16<Sse4_2> {
527 _mm_castps_si128(a.into()).simd_into(token)
528 }
529 );
530 kernel(self, a)
531 }
532 #[inline(always)]
533 fn reinterpret_u32_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
534 crate::kernel!(
535 #[inline(always)]
536 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> u32x4<Sse4_2> {
537 _mm_castps_si128(a.into()).simd_into(token)
538 }
539 );
540 kernel(self, a)
541 }
542 #[inline(always)]
543 fn cvt_u32_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
544 crate::kernel!(
545 #[inline(always)]
546 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> u32x4<Sse4_2> {
547 let mut converted = _mm_cvttps_epi32(a.into());
548 let in_range = _mm_cmplt_ps(a.into(), _mm_set1_ps(2147483648.0));
549 let all_in_range = _mm_movemask_ps(in_range) == 0b1111;
550 if !all_in_range {
551 let excess = _mm_sub_ps(a.into(), _mm_set1_ps(2147483648.0));
552 let excess_converted = _mm_cvttps_epi32(_mm_andnot_ps(in_range, excess));
553 converted = _mm_add_epi32(converted, excess_converted);
554 }
555 converted.simd_into(token)
556 }
557 );
558 kernel(self, a)
559 }
560 #[inline(always)]
561 fn cvt_u32_precise_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
562 crate::kernel!(
563 #[inline(always)]
564 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> u32x4<Sse4_2> {
565 let a = _mm_max_ps(a.into(), _mm_setzero_ps());
566 let mut converted = _mm_cvttps_epi32(a);
567 let in_range = _mm_cmplt_ps(a, _mm_set1_ps(2147483648.0));
568 let all_in_range = _mm_movemask_ps(in_range) == 0b1111;
569 if !all_in_range {
570 let exceeds_unsigned_range =
571 _mm_castps_si128(_mm_cmplt_ps(_mm_set1_ps(4294967040.0), a));
572 let excess = _mm_sub_ps(a, _mm_set1_ps(2147483648.0));
573 let excess_converted = _mm_cvttps_epi32(_mm_andnot_ps(in_range, excess));
574 converted = _mm_add_epi32(converted, excess_converted);
575 converted = _mm_blendv_epi8(
576 converted,
577 _mm_set1_epi32(u32::MAX.cast_signed()),
578 exceeds_unsigned_range,
579 );
580 }
581 converted.simd_into(token)
582 }
583 );
584 kernel(self, a)
585 }
586 #[inline(always)]
587 fn cvt_i32_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
588 crate::kernel!(
589 #[inline(always)]
590 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> i32x4<Sse4_2> {
591 _mm_cvttps_epi32(a.into()).simd_into(token)
592 }
593 );
594 kernel(self, a)
595 }
596 #[inline(always)]
597 fn cvt_i32_precise_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
598 crate::kernel!(
599 #[inline(always)]
600 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> i32x4<Sse4_2> {
601 let a = a.into();
602 let mut converted = _mm_cvttps_epi32(a);
603 let in_range = _mm_cmplt_ps(a, _mm_set1_ps(2147483648.0));
604 let all_in_range = _mm_movemask_ps(in_range) == 0b1111;
605 if !all_in_range {
606 converted = _mm_blendv_epi8(
607 _mm_set1_epi32(i32::MAX),
608 converted,
609 _mm_castps_si128(in_range),
610 );
611 let is_not_nan = _mm_castps_si128(_mm_cmpord_ps(a, a));
612 converted = _mm_and_si128(converted, is_not_nan);
613 }
614 converted.simd_into(token)
615 }
616 );
617 kernel(self, a)
618 }
619 #[inline(always)]
620 fn splat_i8x16(self, val: i8) -> i8x16<Self> {
621 crate::kernel!(
622 #[inline(always)]
623 fn kernel(token: Sse4_2, val: i8) -> i8x16<Sse4_2> {
624 _mm_set1_epi8(val).simd_into(token)
625 }
626 );
627 kernel(self, val)
628 }
629 #[inline(always)]
630 fn load_array_i8x16(self, val: [i8; 16usize]) -> i8x16<Self> {
631 i8x16 {
632 val: crate::transmute::checked_transmute_copy(&val),
633 simd: self,
634 }
635 }
636 #[inline(always)]
637 fn load_array_ref_i8x16(self, val: &[i8; 16usize]) -> i8x16<Self> {
638 i8x16 {
639 val: crate::transmute::checked_transmute_copy(val),
640 simd: self,
641 }
642 }
643 #[inline(always)]
644 fn as_array_i8x16(self, a: i8x16<Self>) -> [i8; 16usize] {
645 crate::transmute::checked_transmute_copy::<__m128i, [i8; 16usize]>(&a.val.0)
646 }
647 #[inline(always)]
648 fn as_array_ref_i8x16(self, a: &i8x16<Self>) -> &[i8; 16usize] {
649 crate::transmute::checked_cast_ref::<__m128i, [i8; 16usize]>(&a.val.0)
650 }
651 #[inline(always)]
652 fn as_array_mut_i8x16(self, a: &mut i8x16<Self>) -> &mut [i8; 16usize] {
653 crate::transmute::checked_cast_mut::<__m128i, [i8; 16usize]>(&mut a.val.0)
654 }
655 #[inline(always)]
656 fn store_array_i8x16(self, a: i8x16<Self>, dest: &mut [i8; 16usize]) -> () {
657 crate::transmute::checked_transmute_store(a.val.0, dest);
658 }
659 #[inline(always)]
660 fn cvt_from_bytes_i8x16(self, a: u8x16<Self>) -> i8x16<Self> {
661 i8x16 {
662 val: crate::transmute::checked_transmute_copy(&a.val),
663 simd: self,
664 }
665 }
666 #[inline(always)]
667 fn cvt_to_bytes_i8x16(self, a: i8x16<Self>) -> u8x16<Self> {
668 u8x16 {
669 val: crate::transmute::checked_transmute_copy(&a.val),
670 simd: self,
671 }
672 }
673 #[inline(always)]
674 fn slide_i8x16<const SHIFT: usize>(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
675 if SHIFT >= 16usize {
676 return b;
677 }
678 let result = dyn_alignr_128(
679 self,
680 self.cvt_to_bytes_i8x16(b).val.0,
681 self.cvt_to_bytes_i8x16(a).val.0,
682 SHIFT,
683 );
684 self.cvt_from_bytes_i8x16(u8x16 {
685 val: crate::support::Aligned128(result),
686 simd: self,
687 })
688 }
689 #[inline(always)]
690 fn slide_within_blocks_i8x16<const SHIFT: usize>(
691 self,
692 a: i8x16<Self>,
693 b: i8x16<Self>,
694 ) -> i8x16<Self> {
695 self.slide_i8x16::<SHIFT>(a, b)
696 }
697 #[inline(always)]
698 fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16<Self>, indices: u8x16<Self>) -> i8x16<Self> {
699 crate::kernel!(
700 #[inline(always)]
701 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, indices: u8x16<Sse4_2>) -> i8x16<Sse4_2> {
702 let result = _mm_shuffle_epi8(token.cvt_to_bytes_i8x16(a).val.0, indices.into());
703 token.cvt_from_bytes_i8x16(u8x16 {
704 val: crate::support::Aligned128(result),
705 simd: token,
706 })
707 }
708 );
709 kernel(self, a, indices)
710 }
711 #[inline(always)]
712 fn add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
713 crate::kernel!(
714 #[inline(always)]
715 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
716 _mm_add_epi8(a.into(), b.into()).simd_into(token)
717 }
718 );
719 kernel(self, a, b)
720 }
721 #[inline(always)]
722 fn sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
723 crate::kernel!(
724 #[inline(always)]
725 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
726 _mm_sub_epi8(a.into(), b.into()).simd_into(token)
727 }
728 );
729 kernel(self, a, b)
730 }
731 #[inline(always)]
732 fn mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
733 crate::kernel!(
734 #[inline(always)]
735 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
736 let dst_even = _mm_mullo_epi16(a.into(), b.into());
737 let dst_odd =
738 _mm_mullo_epi16(_mm_srli_epi16::<8>(a.into()), _mm_srli_epi16::<8>(b.into()));
739 _mm_or_si128(
740 _mm_slli_epi16(dst_odd, 8),
741 _mm_and_si128(dst_even, _mm_set1_epi16(0xFF)),
742 )
743 .simd_into(token)
744 }
745 );
746 kernel(self, a, b)
747 }
748 #[inline(always)]
749 fn and_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
750 crate::kernel!(
751 #[inline(always)]
752 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
753 _mm_and_si128(a.into(), b.into()).simd_into(token)
754 }
755 );
756 kernel(self, a, b)
757 }
758 #[inline(always)]
759 fn or_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
760 crate::kernel!(
761 #[inline(always)]
762 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
763 _mm_or_si128(a.into(), b.into()).simd_into(token)
764 }
765 );
766 kernel(self, a, b)
767 }
768 #[inline(always)]
769 fn xor_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
770 crate::kernel!(
771 #[inline(always)]
772 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
773 _mm_xor_si128(a.into(), b.into()).simd_into(token)
774 }
775 );
776 kernel(self, a, b)
777 }
778 #[inline(always)]
779 fn not_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
780 a ^ !0
781 }
782 #[inline(always)]
783 fn shl_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
784 crate::kernel!(
785 #[inline(always)]
786 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, shift: u32) -> i8x16<Sse4_2> {
787 let val = a.into();
788 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
789 let lo_16 = _mm_unpacklo_epi8(val, _mm_cmpgt_epi8(_mm_setzero_si128(), val));
790 let hi_16 = _mm_unpackhi_epi8(val, _mm_cmpgt_epi8(_mm_setzero_si128(), val));
791 let lo_shifted = _mm_sll_epi16(lo_16, shift_count);
792 let hi_shifted = _mm_sll_epi16(hi_16, shift_count);
793 _mm_packs_epi16(lo_shifted, hi_shifted).simd_into(token)
794 }
795 );
796 kernel(self, a, shift)
797 }
798 #[inline(always)]
799 fn shlv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
800 core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self)
801 }
802 #[inline(always)]
803 fn shr_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
804 crate::kernel!(
805 #[inline(always)]
806 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, shift: u32) -> i8x16<Sse4_2> {
807 let val = a.into();
808 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
809 let lo_16 = _mm_unpacklo_epi8(val, _mm_cmpgt_epi8(_mm_setzero_si128(), val));
810 let hi_16 = _mm_unpackhi_epi8(val, _mm_cmpgt_epi8(_mm_setzero_si128(), val));
811 let lo_shifted = _mm_sra_epi16(lo_16, shift_count);
812 let hi_shifted = _mm_sra_epi16(hi_16, shift_count);
813 _mm_packs_epi16(lo_shifted, hi_shifted).simd_into(token)
814 }
815 );
816 kernel(self, a, shift)
817 }
818 #[inline(always)]
819 fn shrv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
820 core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self)
821 }
822 #[inline(always)]
823 fn simd_eq_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
824 crate::kernel!(
825 #[inline(always)]
826 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> mask8x16<Sse4_2> {
827 _mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
828 }
829 );
830 kernel(self, a, b)
831 }
832 #[inline(always)]
833 fn simd_lt_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
834 crate::kernel!(
835 #[inline(always)]
836 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> mask8x16<Sse4_2> {
837 _mm_cmpgt_epi8(b.into(), a.into()).simd_into(token)
838 }
839 );
840 kernel(self, a, b)
841 }
842 #[inline(always)]
843 fn simd_le_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
844 crate::kernel!(
845 #[inline(always)]
846 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> mask8x16<Sse4_2> {
847 _mm_cmpeq_epi8(_mm_min_epi8(a.into(), b.into()), a.into()).simd_into(token)
848 }
849 );
850 kernel(self, a, b)
851 }
852 #[inline(always)]
853 fn simd_ge_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
854 crate::kernel!(
855 #[inline(always)]
856 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> mask8x16<Sse4_2> {
857 _mm_cmpeq_epi8(_mm_max_epi8(a.into(), b.into()), a.into()).simd_into(token)
858 }
859 );
860 kernel(self, a, b)
861 }
862 #[inline(always)]
863 fn simd_gt_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
864 crate::kernel!(
865 #[inline(always)]
866 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> mask8x16<Sse4_2> {
867 _mm_cmpgt_epi8(a.into(), b.into()).simd_into(token)
868 }
869 );
870 kernel(self, a, b)
871 }
872 #[inline(always)]
873 fn zip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
874 crate::kernel!(
875 #[inline(always)]
876 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
877 _mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
878 }
879 );
880 kernel(self, a, b)
881 }
882 #[inline(always)]
883 fn zip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
884 crate::kernel!(
885 #[inline(always)]
886 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
887 _mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
888 }
889 );
890 kernel(self, a, b)
891 }
892 #[inline(always)]
893 fn unzip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
894 crate::kernel!(
895 #[inline(always)]
896 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
897 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
898 let t1 = _mm_shuffle_epi8(a.into(), mask);
899 let t2 = _mm_shuffle_epi8(b.into(), mask);
900 _mm_unpacklo_epi64(t1, t2).simd_into(token)
901 }
902 );
903 kernel(self, a, b)
904 }
905 #[inline(always)]
906 fn unzip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
907 crate::kernel!(
908 #[inline(always)]
909 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
910 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
911 let t1 = _mm_shuffle_epi8(a.into(), mask);
912 let t2 = _mm_shuffle_epi8(b.into(), mask);
913 _mm_unpackhi_epi64(t1, t2).simd_into(token)
914 }
915 );
916 kernel(self, a, b)
917 }
918 #[inline(always)]
919 fn interleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
920 (self.zip_low_i8x16(a, b), self.zip_high_i8x16(a, b))
921 }
922 #[inline(always)]
923 fn deinterleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
924 (self.unzip_low_i8x16(a, b), self.unzip_high_i8x16(a, b))
925 }
926 #[inline(always)]
927 fn select_i8x16(self, a: mask8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
928 crate::kernel!(
929 #[inline(always)]
930 fn kernel(
931 token: Sse4_2,
932 a: mask8x16<Sse4_2>,
933 b: i8x16<Sse4_2>,
934 c: i8x16<Sse4_2>,
935 ) -> i8x16<Sse4_2> {
936 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
937 }
938 );
939 kernel(self, a, b, c)
940 }
941 #[inline(always)]
942 fn min_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
943 crate::kernel!(
944 #[inline(always)]
945 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
946 _mm_min_epi8(a.into(), b.into()).simd_into(token)
947 }
948 );
949 kernel(self, a, b)
950 }
951 #[inline(always)]
952 fn max_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
953 crate::kernel!(
954 #[inline(always)]
955 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
956 _mm_max_epi8(a.into(), b.into()).simd_into(token)
957 }
958 );
959 kernel(self, a, b)
960 }
961 #[inline(always)]
962 fn combine_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x32<Self> {
963 i8x32 {
964 val: crate::support::Aligned256([a.val.0, b.val.0]),
965 simd: self,
966 }
967 }
968 #[inline(always)]
969 fn neg_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
970 crate::kernel!(
971 #[inline(always)]
972 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
973 _mm_sub_epi8(_mm_setzero_si128(), a.into()).simd_into(token)
974 }
975 );
976 kernel(self, a)
977 }
978 #[inline(always)]
979 fn reinterpret_u8_i8x16(self, a: i8x16<Self>) -> u8x16<Self> {
980 crate::kernel!(
981 #[inline(always)]
982 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>) -> u8x16<Sse4_2> {
983 __m128i::from(a).simd_into(token)
984 }
985 );
986 kernel(self, a)
987 }
988 #[inline(always)]
989 fn reinterpret_u32_i8x16(self, a: i8x16<Self>) -> u32x4<Self> {
990 crate::kernel!(
991 #[inline(always)]
992 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>) -> u32x4<Sse4_2> {
993 __m128i::from(a).simd_into(token)
994 }
995 );
996 kernel(self, a)
997 }
998 #[inline(always)]
999 fn splat_u8x16(self, val: u8) -> u8x16<Self> {
1000 crate::kernel!(
1001 #[inline(always)]
1002 fn kernel(token: Sse4_2, val: u8) -> u8x16<Sse4_2> {
1003 _mm_set1_epi8(val.cast_signed()).simd_into(token)
1004 }
1005 );
1006 kernel(self, val)
1007 }
1008 #[inline(always)]
1009 fn load_array_u8x16(self, val: [u8; 16usize]) -> u8x16<Self> {
1010 u8x16 {
1011 val: crate::transmute::checked_transmute_copy(&val),
1012 simd: self,
1013 }
1014 }
1015 #[inline(always)]
1016 fn load_array_ref_u8x16(self, val: &[u8; 16usize]) -> u8x16<Self> {
1017 u8x16 {
1018 val: crate::transmute::checked_transmute_copy(val),
1019 simd: self,
1020 }
1021 }
1022 #[inline(always)]
1023 fn as_array_u8x16(self, a: u8x16<Self>) -> [u8; 16usize] {
1024 crate::transmute::checked_transmute_copy::<__m128i, [u8; 16usize]>(&a.val.0)
1025 }
1026 #[inline(always)]
1027 fn as_array_ref_u8x16(self, a: &u8x16<Self>) -> &[u8; 16usize] {
1028 crate::transmute::checked_cast_ref::<__m128i, [u8; 16usize]>(&a.val.0)
1029 }
1030 #[inline(always)]
1031 fn as_array_mut_u8x16(self, a: &mut u8x16<Self>) -> &mut [u8; 16usize] {
1032 crate::transmute::checked_cast_mut::<__m128i, [u8; 16usize]>(&mut a.val.0)
1033 }
1034 #[inline(always)]
1035 fn store_array_u8x16(self, a: u8x16<Self>, dest: &mut [u8; 16usize]) -> () {
1036 crate::transmute::checked_transmute_store(a.val.0, dest);
1037 }
1038 #[inline(always)]
1039 fn cvt_from_bytes_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1040 u8x16 {
1041 val: crate::transmute::checked_transmute_copy(&a.val),
1042 simd: self,
1043 }
1044 }
1045 #[inline(always)]
1046 fn cvt_to_bytes_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1047 u8x16 {
1048 val: crate::transmute::checked_transmute_copy(&a.val),
1049 simd: self,
1050 }
1051 }
1052 #[inline(always)]
1053 fn slide_u8x16<const SHIFT: usize>(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1054 if SHIFT >= 16usize {
1055 return b;
1056 }
1057 let result = dyn_alignr_128(
1058 self,
1059 self.cvt_to_bytes_u8x16(b).val.0,
1060 self.cvt_to_bytes_u8x16(a).val.0,
1061 SHIFT,
1062 );
1063 self.cvt_from_bytes_u8x16(u8x16 {
1064 val: crate::support::Aligned128(result),
1065 simd: self,
1066 })
1067 }
1068 #[inline(always)]
1069 fn slide_within_blocks_u8x16<const SHIFT: usize>(
1070 self,
1071 a: u8x16<Self>,
1072 b: u8x16<Self>,
1073 ) -> u8x16<Self> {
1074 self.slide_u8x16::<SHIFT>(a, b)
1075 }
1076 #[inline(always)]
1077 fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1078 crate::kernel!(
1079 #[inline(always)]
1080 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, indices: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1081 let result = _mm_shuffle_epi8(token.cvt_to_bytes_u8x16(a).val.0, indices.into());
1082 token.cvt_from_bytes_u8x16(u8x16 {
1083 val: crate::support::Aligned128(result),
1084 simd: token,
1085 })
1086 }
1087 );
1088 kernel(self, a, indices)
1089 }
1090 #[inline(always)]
1091 fn add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1092 crate::kernel!(
1093 #[inline(always)]
1094 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1095 _mm_add_epi8(a.into(), b.into()).simd_into(token)
1096 }
1097 );
1098 kernel(self, a, b)
1099 }
1100 #[inline(always)]
1101 fn sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1102 crate::kernel!(
1103 #[inline(always)]
1104 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1105 _mm_sub_epi8(a.into(), b.into()).simd_into(token)
1106 }
1107 );
1108 kernel(self, a, b)
1109 }
1110 #[inline(always)]
1111 fn mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1112 crate::kernel!(
1113 #[inline(always)]
1114 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1115 let dst_even = _mm_mullo_epi16(a.into(), b.into());
1116 let dst_odd =
1117 _mm_mullo_epi16(_mm_srli_epi16::<8>(a.into()), _mm_srli_epi16::<8>(b.into()));
1118 _mm_or_si128(
1119 _mm_slli_epi16(dst_odd, 8),
1120 _mm_and_si128(dst_even, _mm_set1_epi16(0xFF)),
1121 )
1122 .simd_into(token)
1123 }
1124 );
1125 kernel(self, a, b)
1126 }
1127 #[inline(always)]
1128 fn and_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1129 crate::kernel!(
1130 #[inline(always)]
1131 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1132 _mm_and_si128(a.into(), b.into()).simd_into(token)
1133 }
1134 );
1135 kernel(self, a, b)
1136 }
1137 #[inline(always)]
1138 fn or_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1139 crate::kernel!(
1140 #[inline(always)]
1141 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1142 _mm_or_si128(a.into(), b.into()).simd_into(token)
1143 }
1144 );
1145 kernel(self, a, b)
1146 }
1147 #[inline(always)]
1148 fn xor_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1149 crate::kernel!(
1150 #[inline(always)]
1151 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1152 _mm_xor_si128(a.into(), b.into()).simd_into(token)
1153 }
1154 );
1155 kernel(self, a, b)
1156 }
1157 #[inline(always)]
1158 fn not_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1159 a ^ !0
1160 }
1161 #[inline(always)]
1162 fn shl_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
1163 crate::kernel!(
1164 #[inline(always)]
1165 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, shift: u32) -> u8x16<Sse4_2> {
1166 let val = a.into();
1167 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1168 let lo_16 = _mm_unpacklo_epi8(val, _mm_setzero_si128());
1169 let hi_16 = _mm_unpackhi_epi8(val, _mm_setzero_si128());
1170 let lo_shifted = _mm_sll_epi16(lo_16, shift_count);
1171 let hi_shifted = _mm_sll_epi16(hi_16, shift_count);
1172 _mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
1173 }
1174 );
1175 kernel(self, a, shift)
1176 }
1177 #[inline(always)]
1178 fn shlv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1179 core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self)
1180 }
1181 #[inline(always)]
1182 fn shr_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
1183 crate::kernel!(
1184 #[inline(always)]
1185 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, shift: u32) -> u8x16<Sse4_2> {
1186 let val = a.into();
1187 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1188 let lo_16 = _mm_unpacklo_epi8(val, _mm_setzero_si128());
1189 let hi_16 = _mm_unpackhi_epi8(val, _mm_setzero_si128());
1190 let lo_shifted = _mm_srl_epi16(lo_16, shift_count);
1191 let hi_shifted = _mm_srl_epi16(hi_16, shift_count);
1192 _mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
1193 }
1194 );
1195 kernel(self, a, shift)
1196 }
1197 #[inline(always)]
1198 fn shrv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1199 core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self)
1200 }
1201 #[inline(always)]
1202 fn simd_eq_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
1203 crate::kernel!(
1204 #[inline(always)]
1205 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> mask8x16<Sse4_2> {
1206 _mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
1207 }
1208 );
1209 kernel(self, a, b)
1210 }
1211 #[inline(always)]
1212 fn simd_lt_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
1213 crate::kernel!(
1214 #[inline(always)]
1215 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> mask8x16<Sse4_2> {
1216 let sign_bit = _mm_set1_epi8(0x80u8.cast_signed());
1217 let a_signed = _mm_xor_si128(a.into(), sign_bit);
1218 let b_signed = _mm_xor_si128(b.into(), sign_bit);
1219 _mm_cmpgt_epi8(b_signed, a_signed).simd_into(token)
1220 }
1221 );
1222 kernel(self, a, b)
1223 }
1224 #[inline(always)]
1225 fn simd_le_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
1226 crate::kernel!(
1227 #[inline(always)]
1228 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> mask8x16<Sse4_2> {
1229 _mm_cmpeq_epi8(_mm_min_epu8(a.into(), b.into()), a.into()).simd_into(token)
1230 }
1231 );
1232 kernel(self, a, b)
1233 }
1234 #[inline(always)]
1235 fn simd_ge_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
1236 crate::kernel!(
1237 #[inline(always)]
1238 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> mask8x16<Sse4_2> {
1239 _mm_cmpeq_epi8(_mm_max_epu8(a.into(), b.into()), a.into()).simd_into(token)
1240 }
1241 );
1242 kernel(self, a, b)
1243 }
1244 #[inline(always)]
1245 fn simd_gt_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
1246 crate::kernel!(
1247 #[inline(always)]
1248 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> mask8x16<Sse4_2> {
1249 let sign_bit = _mm_set1_epi8(0x80u8.cast_signed());
1250 let a_signed = _mm_xor_si128(a.into(), sign_bit);
1251 let b_signed = _mm_xor_si128(b.into(), sign_bit);
1252 _mm_cmpgt_epi8(a_signed, b_signed).simd_into(token)
1253 }
1254 );
1255 kernel(self, a, b)
1256 }
1257 #[inline(always)]
1258 fn zip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1259 crate::kernel!(
1260 #[inline(always)]
1261 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1262 _mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
1263 }
1264 );
1265 kernel(self, a, b)
1266 }
1267 #[inline(always)]
1268 fn zip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1269 crate::kernel!(
1270 #[inline(always)]
1271 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1272 _mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
1273 }
1274 );
1275 kernel(self, a, b)
1276 }
1277 #[inline(always)]
1278 fn unzip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1279 crate::kernel!(
1280 #[inline(always)]
1281 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1282 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
1283 let t1 = _mm_shuffle_epi8(a.into(), mask);
1284 let t2 = _mm_shuffle_epi8(b.into(), mask);
1285 _mm_unpacklo_epi64(t1, t2).simd_into(token)
1286 }
1287 );
1288 kernel(self, a, b)
1289 }
1290 #[inline(always)]
1291 fn unzip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1292 crate::kernel!(
1293 #[inline(always)]
1294 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1295 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
1296 let t1 = _mm_shuffle_epi8(a.into(), mask);
1297 let t2 = _mm_shuffle_epi8(b.into(), mask);
1298 _mm_unpackhi_epi64(t1, t2).simd_into(token)
1299 }
1300 );
1301 kernel(self, a, b)
1302 }
1303 #[inline(always)]
1304 fn interleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
1305 (self.zip_low_u8x16(a, b), self.zip_high_u8x16(a, b))
1306 }
1307 #[inline(always)]
1308 fn deinterleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
1309 (self.unzip_low_u8x16(a, b), self.unzip_high_u8x16(a, b))
1310 }
1311 #[inline(always)]
1312 fn select_u8x16(self, a: mask8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
1313 crate::kernel!(
1314 #[inline(always)]
1315 fn kernel(
1316 token: Sse4_2,
1317 a: mask8x16<Sse4_2>,
1318 b: u8x16<Sse4_2>,
1319 c: u8x16<Sse4_2>,
1320 ) -> u8x16<Sse4_2> {
1321 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
1322 }
1323 );
1324 kernel(self, a, b, c)
1325 }
1326 #[inline(always)]
1327 fn min_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1328 crate::kernel!(
1329 #[inline(always)]
1330 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1331 _mm_min_epu8(a.into(), b.into()).simd_into(token)
1332 }
1333 );
1334 kernel(self, a, b)
1335 }
1336 #[inline(always)]
1337 fn max_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1338 crate::kernel!(
1339 #[inline(always)]
1340 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1341 _mm_max_epu8(a.into(), b.into()).simd_into(token)
1342 }
1343 );
1344 kernel(self, a, b)
1345 }
1346 #[inline(always)]
1347 fn combine_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x32<Self> {
1348 u8x32 {
1349 val: crate::support::Aligned256([a.val.0, b.val.0]),
1350 simd: self,
1351 }
1352 }
1353 #[inline(always)]
1354 fn widen_u8x16(self, a: u8x16<Self>) -> u16x16<Self> {
1355 crate::kernel!(
1356 #[inline(always)]
1357 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>) -> u16x16<Sse4_2> {
1358 let raw = a.into();
1359 let high = _mm_cvtepu8_epi16(raw).simd_into(token);
1360 let low = _mm_cvtepu8_epi16(_mm_srli_si128::<8>(raw)).simd_into(token);
1361 token.combine_u16x8(high, low)
1362 }
1363 );
1364 kernel(self, a)
1365 }
1366 #[inline(always)]
1367 fn reinterpret_u32_u8x16(self, a: u8x16<Self>) -> u32x4<Self> {
1368 crate::kernel!(
1369 #[inline(always)]
1370 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>) -> u32x4<Sse4_2> {
1371 __m128i::from(a).simd_into(token)
1372 }
1373 );
1374 kernel(self, a)
1375 }
1376 #[inline(always)]
1377 fn splat_mask8x16(self, val: bool) -> mask8x16<Self> {
1378 crate::kernel!(
1379 #[inline(always)]
1380 fn kernel(token: Sse4_2, val: bool) -> mask8x16<Sse4_2> {
1381 let val: i8 = if val { !0 } else { 0 };
1382 _mm_set1_epi8(val).simd_into(token)
1383 }
1384 );
1385 kernel(self, val)
1386 }
1387 #[inline(always)]
1388 fn load_array_mask8x16(self, val: [i8; 16usize]) -> mask8x16<Self> {
1389 mask8x16 {
1390 val: crate::transmute::checked_transmute_copy(&val),
1391 simd: self,
1392 }
1393 }
1394 #[inline(always)]
1395 fn as_array_mask8x16(self, a: mask8x16<Self>) -> [i8; 16usize] {
1396 crate::transmute::checked_transmute_copy::<__m128i, [i8; 16usize]>(&a.val.0)
1397 }
1398 #[inline(always)]
1399 fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16<Self> {
1400 crate::kernel!(
1401 #[inline(always)]
1402 fn kernel(token: Sse4_2, bits: u64) -> mask8x16<Sse4_2> {
1403 {
1404 let bit_bytes = _mm_cvtsi32_si128(bits as i32);
1405 let bit_bytes = _mm_shuffle_epi8(
1406 bit_bytes,
1407 _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1),
1408 );
1409 let bit_mask =
1410 _mm_setr_epi8(1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128);
1411 _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask)
1412 }
1413 .simd_into(token)
1414 }
1415 );
1416 kernel(self, bits)
1417 }
1418 #[inline(always)]
1419 fn to_bitmask_mask8x16(self, a: mask8x16<Self>) -> u64 {
1420 crate::kernel!(
1421 #[inline(always)]
1422 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>) -> u64 {
1423 _mm_movemask_epi8(a.into()) as u32 as u64
1424 }
1425 );
1426 kernel(self, a)
1427 }
1428 #[inline(always)]
1429 fn set_mask8x16(self, a: &mut mask8x16<Self>, index: usize, value: bool) -> () {
1430 assert!(
1431 index < 16usize,
1432 "mask lane index {index} is out of bounds for {} lanes",
1433 16usize
1434 );
1435 let mut lanes = self.as_array_mask8x16(*a);
1436 lanes[index] = if value { !0 } else { 0 };
1437 *a = self.load_array_mask8x16(lanes);
1438 }
1439 #[inline(always)]
1440 fn and_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
1441 crate::kernel!(
1442 #[inline(always)]
1443 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>, b: mask8x16<Sse4_2>) -> mask8x16<Sse4_2> {
1444 _mm_and_si128(a.into(), b.into()).simd_into(token)
1445 }
1446 );
1447 kernel(self, a, b)
1448 }
1449 #[inline(always)]
1450 fn or_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
1451 crate::kernel!(
1452 #[inline(always)]
1453 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>, b: mask8x16<Sse4_2>) -> mask8x16<Sse4_2> {
1454 _mm_or_si128(a.into(), b.into()).simd_into(token)
1455 }
1456 );
1457 kernel(self, a, b)
1458 }
1459 #[inline(always)]
1460 fn xor_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
1461 crate::kernel!(
1462 #[inline(always)]
1463 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>, b: mask8x16<Sse4_2>) -> mask8x16<Sse4_2> {
1464 _mm_xor_si128(a.into(), b.into()).simd_into(token)
1465 }
1466 );
1467 kernel(self, a, b)
1468 }
1469 #[inline(always)]
1470 fn not_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
1471 self.xor_mask8x16(a, self.splat_mask8x16(true))
1472 }
1473 #[inline(always)]
1474 fn select_mask8x16(
1475 self,
1476 a: mask8x16<Self>,
1477 b: mask8x16<Self>,
1478 c: mask8x16<Self>,
1479 ) -> mask8x16<Self> {
1480 crate::kernel!(
1481 #[inline(always)]
1482 fn kernel(
1483 token: Sse4_2,
1484 a: mask8x16<Sse4_2>,
1485 b: mask8x16<Sse4_2>,
1486 c: mask8x16<Sse4_2>,
1487 ) -> mask8x16<Sse4_2> {
1488 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
1489 }
1490 );
1491 kernel(self, a, b, c)
1492 }
1493 #[inline(always)]
1494 fn simd_eq_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
1495 crate::kernel!(
1496 #[inline(always)]
1497 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>, b: mask8x16<Sse4_2>) -> mask8x16<Sse4_2> {
1498 _mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
1499 }
1500 );
1501 kernel(self, a, b)
1502 }
1503 #[inline(always)]
1504 fn any_true_mask8x16(self, a: mask8x16<Self>) -> bool {
1505 crate::kernel!(
1506 #[inline(always)]
1507 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>) -> bool {
1508 _mm_movemask_epi8(a.into()) as u32 != 0
1509 }
1510 );
1511 kernel(self, a)
1512 }
1513 #[inline(always)]
1514 fn all_true_mask8x16(self, a: mask8x16<Self>) -> bool {
1515 crate::kernel!(
1516 #[inline(always)]
1517 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>) -> bool {
1518 _mm_movemask_epi8(a.into()) as u32 == 0xffff
1519 }
1520 );
1521 kernel(self, a)
1522 }
1523 #[inline(always)]
1524 fn any_false_mask8x16(self, a: mask8x16<Self>) -> bool {
1525 crate::kernel!(
1526 #[inline(always)]
1527 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>) -> bool {
1528 _mm_movemask_epi8(a.into()) as u32 != 0xffff
1529 }
1530 );
1531 kernel(self, a)
1532 }
1533 #[inline(always)]
1534 fn all_false_mask8x16(self, a: mask8x16<Self>) -> bool {
1535 crate::kernel!(
1536 #[inline(always)]
1537 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>) -> bool {
1538 _mm_movemask_epi8(a.into()) as u32 == 0
1539 }
1540 );
1541 kernel(self, a)
1542 }
1543 #[inline(always)]
1544 fn combine_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x32<Self> {
1545 mask8x32 {
1546 val: crate::support::Aligned256([a.val.0, b.val.0]),
1547 simd: self,
1548 }
1549 }
1550 #[inline(always)]
1551 fn splat_i16x8(self, val: i16) -> i16x8<Self> {
1552 crate::kernel!(
1553 #[inline(always)]
1554 fn kernel(token: Sse4_2, val: i16) -> i16x8<Sse4_2> {
1555 _mm_set1_epi16(val).simd_into(token)
1556 }
1557 );
1558 kernel(self, val)
1559 }
1560 #[inline(always)]
1561 fn load_array_i16x8(self, val: [i16; 8usize]) -> i16x8<Self> {
1562 i16x8 {
1563 val: crate::transmute::checked_transmute_copy(&val),
1564 simd: self,
1565 }
1566 }
1567 #[inline(always)]
1568 fn load_array_ref_i16x8(self, val: &[i16; 8usize]) -> i16x8<Self> {
1569 i16x8 {
1570 val: crate::transmute::checked_transmute_copy(val),
1571 simd: self,
1572 }
1573 }
1574 #[inline(always)]
1575 fn as_array_i16x8(self, a: i16x8<Self>) -> [i16; 8usize] {
1576 crate::transmute::checked_transmute_copy::<__m128i, [i16; 8usize]>(&a.val.0)
1577 }
1578 #[inline(always)]
1579 fn as_array_ref_i16x8(self, a: &i16x8<Self>) -> &[i16; 8usize] {
1580 crate::transmute::checked_cast_ref::<__m128i, [i16; 8usize]>(&a.val.0)
1581 }
1582 #[inline(always)]
1583 fn as_array_mut_i16x8(self, a: &mut i16x8<Self>) -> &mut [i16; 8usize] {
1584 crate::transmute::checked_cast_mut::<__m128i, [i16; 8usize]>(&mut a.val.0)
1585 }
1586 #[inline(always)]
1587 fn store_array_i16x8(self, a: i16x8<Self>, dest: &mut [i16; 8usize]) -> () {
1588 crate::transmute::checked_transmute_store(a.val.0, dest);
1589 }
1590 #[inline(always)]
1591 fn cvt_from_bytes_i16x8(self, a: u8x16<Self>) -> i16x8<Self> {
1592 i16x8 {
1593 val: crate::transmute::checked_transmute_copy(&a.val),
1594 simd: self,
1595 }
1596 }
1597 #[inline(always)]
1598 fn cvt_to_bytes_i16x8(self, a: i16x8<Self>) -> u8x16<Self> {
1599 u8x16 {
1600 val: crate::transmute::checked_transmute_copy(&a.val),
1601 simd: self,
1602 }
1603 }
1604 #[inline(always)]
1605 fn slide_i16x8<const SHIFT: usize>(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1606 if SHIFT >= 8usize {
1607 return b;
1608 }
1609 let result = dyn_alignr_128(
1610 self,
1611 self.cvt_to_bytes_i16x8(b).val.0,
1612 self.cvt_to_bytes_i16x8(a).val.0,
1613 SHIFT * 2usize,
1614 );
1615 self.cvt_from_bytes_i16x8(u8x16 {
1616 val: crate::support::Aligned128(result),
1617 simd: self,
1618 })
1619 }
1620 #[inline(always)]
1621 fn slide_within_blocks_i16x8<const SHIFT: usize>(
1622 self,
1623 a: i16x8<Self>,
1624 b: i16x8<Self>,
1625 ) -> i16x8<Self> {
1626 self.slide_i16x8::<SHIFT>(a, b)
1627 }
1628 #[inline(always)]
1629 fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8<Self>, indices: u8x16<Self>) -> i16x8<Self> {
1630 crate::kernel!(
1631 #[inline(always)]
1632 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, indices: u8x16<Sse4_2>) -> i16x8<Sse4_2> {
1633 let result = _mm_shuffle_epi8(token.cvt_to_bytes_i16x8(a).val.0, indices.into());
1634 token.cvt_from_bytes_i16x8(u8x16 {
1635 val: crate::support::Aligned128(result),
1636 simd: token,
1637 })
1638 }
1639 );
1640 kernel(self, a, indices)
1641 }
1642 #[inline(always)]
1643 fn add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1644 crate::kernel!(
1645 #[inline(always)]
1646 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
1647 _mm_add_epi16(a.into(), b.into()).simd_into(token)
1648 }
1649 );
1650 kernel(self, a, b)
1651 }
1652 #[inline(always)]
1653 fn sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1654 crate::kernel!(
1655 #[inline(always)]
1656 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
1657 _mm_sub_epi16(a.into(), b.into()).simd_into(token)
1658 }
1659 );
1660 kernel(self, a, b)
1661 }
1662 #[inline(always)]
1663 fn mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1664 crate::kernel!(
1665 #[inline(always)]
1666 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
1667 _mm_mullo_epi16(a.into(), b.into()).simd_into(token)
1668 }
1669 );
1670 kernel(self, a, b)
1671 }
1672 #[inline(always)]
1673 fn and_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1674 crate::kernel!(
1675 #[inline(always)]
1676 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
1677 _mm_and_si128(a.into(), b.into()).simd_into(token)
1678 }
1679 );
1680 kernel(self, a, b)
1681 }
1682 #[inline(always)]
1683 fn or_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1684 crate::kernel!(
1685 #[inline(always)]
1686 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
1687 _mm_or_si128(a.into(), b.into()).simd_into(token)
1688 }
1689 );
1690 kernel(self, a, b)
1691 }
1692 #[inline(always)]
1693 fn xor_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1694 crate::kernel!(
1695 #[inline(always)]
1696 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
1697 _mm_xor_si128(a.into(), b.into()).simd_into(token)
1698 }
1699 );
1700 kernel(self, a, b)
1701 }
1702 #[inline(always)]
1703 fn not_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
1704 a ^ !0
1705 }
1706 #[inline(always)]
1707 fn shl_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
1708 crate::kernel!(
1709 #[inline(always)]
1710 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, shift: u32) -> i16x8<Sse4_2> {
1711 _mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
1712 }
1713 );
1714 kernel(self, a, shift)
1715 }
1716 #[inline(always)]
1717 fn shlv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1718 core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self)
1719 }
1720 #[inline(always)]
1721 fn shr_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
1722 crate::kernel!(
1723 #[inline(always)]
1724 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, shift: u32) -> i16x8<Sse4_2> {
1725 _mm_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
1726 }
1727 );
1728 kernel(self, a, shift)
1729 }
1730 #[inline(always)]
1731 fn shrv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1732 core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self)
1733 }
1734 #[inline(always)]
1735 fn simd_eq_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
1736 crate::kernel!(
1737 #[inline(always)]
1738 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> mask16x8<Sse4_2> {
1739 _mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
1740 }
1741 );
1742 kernel(self, a, b)
1743 }
1744 #[inline(always)]
1745 fn simd_lt_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
1746 crate::kernel!(
1747 #[inline(always)]
1748 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> mask16x8<Sse4_2> {
1749 _mm_cmpgt_epi16(b.into(), a.into()).simd_into(token)
1750 }
1751 );
1752 kernel(self, a, b)
1753 }
1754 #[inline(always)]
1755 fn simd_le_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
1756 crate::kernel!(
1757 #[inline(always)]
1758 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> mask16x8<Sse4_2> {
1759 _mm_cmpeq_epi16(_mm_min_epi16(a.into(), b.into()), a.into()).simd_into(token)
1760 }
1761 );
1762 kernel(self, a, b)
1763 }
1764 #[inline(always)]
1765 fn simd_ge_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
1766 crate::kernel!(
1767 #[inline(always)]
1768 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> mask16x8<Sse4_2> {
1769 _mm_cmpeq_epi16(_mm_max_epi16(a.into(), b.into()), a.into()).simd_into(token)
1770 }
1771 );
1772 kernel(self, a, b)
1773 }
1774 #[inline(always)]
1775 fn simd_gt_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
1776 crate::kernel!(
1777 #[inline(always)]
1778 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> mask16x8<Sse4_2> {
1779 _mm_cmpgt_epi16(a.into(), b.into()).simd_into(token)
1780 }
1781 );
1782 kernel(self, a, b)
1783 }
1784 #[inline(always)]
1785 fn zip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1786 crate::kernel!(
1787 #[inline(always)]
1788 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
1789 _mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
1790 }
1791 );
1792 kernel(self, a, b)
1793 }
1794 #[inline(always)]
1795 fn zip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1796 crate::kernel!(
1797 #[inline(always)]
1798 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
1799 _mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
1800 }
1801 );
1802 kernel(self, a, b)
1803 }
1804 #[inline(always)]
1805 fn unzip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1806 crate::kernel!(
1807 #[inline(always)]
1808 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
1809 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
1810 let t1 = _mm_shuffle_epi8(a.into(), mask);
1811 let t2 = _mm_shuffle_epi8(b.into(), mask);
1812 _mm_unpacklo_epi64(t1, t2).simd_into(token)
1813 }
1814 );
1815 kernel(self, a, b)
1816 }
1817 #[inline(always)]
1818 fn unzip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1819 crate::kernel!(
1820 #[inline(always)]
1821 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
1822 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
1823 let t1 = _mm_shuffle_epi8(a.into(), mask);
1824 let t2 = _mm_shuffle_epi8(b.into(), mask);
1825 _mm_unpackhi_epi64(t1, t2).simd_into(token)
1826 }
1827 );
1828 kernel(self, a, b)
1829 }
1830 #[inline(always)]
1831 fn interleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
1832 (self.zip_low_i16x8(a, b), self.zip_high_i16x8(a, b))
1833 }
1834 #[inline(always)]
1835 fn deinterleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
1836 (self.unzip_low_i16x8(a, b), self.unzip_high_i16x8(a, b))
1837 }
1838 #[inline(always)]
1839 fn select_i16x8(self, a: mask16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
1840 crate::kernel!(
1841 #[inline(always)]
1842 fn kernel(
1843 token: Sse4_2,
1844 a: mask16x8<Sse4_2>,
1845 b: i16x8<Sse4_2>,
1846 c: i16x8<Sse4_2>,
1847 ) -> i16x8<Sse4_2> {
1848 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
1849 }
1850 );
1851 kernel(self, a, b, c)
1852 }
1853 #[inline(always)]
1854 fn min_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1855 crate::kernel!(
1856 #[inline(always)]
1857 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
1858 _mm_min_epi16(a.into(), b.into()).simd_into(token)
1859 }
1860 );
1861 kernel(self, a, b)
1862 }
1863 #[inline(always)]
1864 fn max_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
1865 crate::kernel!(
1866 #[inline(always)]
1867 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
1868 _mm_max_epi16(a.into(), b.into()).simd_into(token)
1869 }
1870 );
1871 kernel(self, a, b)
1872 }
1873 #[inline(always)]
1874 fn combine_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x16<Self> {
1875 i16x16 {
1876 val: crate::support::Aligned256([a.val.0, b.val.0]),
1877 simd: self,
1878 }
1879 }
1880 #[inline(always)]
1881 fn neg_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
1882 crate::kernel!(
1883 #[inline(always)]
1884 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
1885 _mm_sub_epi16(_mm_setzero_si128(), a.into()).simd_into(token)
1886 }
1887 );
1888 kernel(self, a)
1889 }
1890 #[inline(always)]
1891 fn reinterpret_u8_i16x8(self, a: i16x8<Self>) -> u8x16<Self> {
1892 crate::kernel!(
1893 #[inline(always)]
1894 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>) -> u8x16<Sse4_2> {
1895 __m128i::from(a).simd_into(token)
1896 }
1897 );
1898 kernel(self, a)
1899 }
1900 #[inline(always)]
1901 fn reinterpret_u32_i16x8(self, a: i16x8<Self>) -> u32x4<Self> {
1902 crate::kernel!(
1903 #[inline(always)]
1904 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>) -> u32x4<Sse4_2> {
1905 __m128i::from(a).simd_into(token)
1906 }
1907 );
1908 kernel(self, a)
1909 }
1910 #[inline(always)]
1911 fn splat_u16x8(self, val: u16) -> u16x8<Self> {
1912 crate::kernel!(
1913 #[inline(always)]
1914 fn kernel(token: Sse4_2, val: u16) -> u16x8<Sse4_2> {
1915 _mm_set1_epi16(val.cast_signed()).simd_into(token)
1916 }
1917 );
1918 kernel(self, val)
1919 }
1920 #[inline(always)]
1921 fn load_array_u16x8(self, val: [u16; 8usize]) -> u16x8<Self> {
1922 u16x8 {
1923 val: crate::transmute::checked_transmute_copy(&val),
1924 simd: self,
1925 }
1926 }
1927 #[inline(always)]
1928 fn load_array_ref_u16x8(self, val: &[u16; 8usize]) -> u16x8<Self> {
1929 u16x8 {
1930 val: crate::transmute::checked_transmute_copy(val),
1931 simd: self,
1932 }
1933 }
1934 #[inline(always)]
1935 fn as_array_u16x8(self, a: u16x8<Self>) -> [u16; 8usize] {
1936 crate::transmute::checked_transmute_copy::<__m128i, [u16; 8usize]>(&a.val.0)
1937 }
1938 #[inline(always)]
1939 fn as_array_ref_u16x8(self, a: &u16x8<Self>) -> &[u16; 8usize] {
1940 crate::transmute::checked_cast_ref::<__m128i, [u16; 8usize]>(&a.val.0)
1941 }
1942 #[inline(always)]
1943 fn as_array_mut_u16x8(self, a: &mut u16x8<Self>) -> &mut [u16; 8usize] {
1944 crate::transmute::checked_cast_mut::<__m128i, [u16; 8usize]>(&mut a.val.0)
1945 }
1946 #[inline(always)]
1947 fn store_array_u16x8(self, a: u16x8<Self>, dest: &mut [u16; 8usize]) -> () {
1948 crate::transmute::checked_transmute_store(a.val.0, dest);
1949 }
1950 #[inline(always)]
1951 fn cvt_from_bytes_u16x8(self, a: u8x16<Self>) -> u16x8<Self> {
1952 u16x8 {
1953 val: crate::transmute::checked_transmute_copy(&a.val),
1954 simd: self,
1955 }
1956 }
1957 #[inline(always)]
1958 fn cvt_to_bytes_u16x8(self, a: u16x8<Self>) -> u8x16<Self> {
1959 u8x16 {
1960 val: crate::transmute::checked_transmute_copy(&a.val),
1961 simd: self,
1962 }
1963 }
1964 #[inline(always)]
1965 fn slide_u16x8<const SHIFT: usize>(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
1966 if SHIFT >= 8usize {
1967 return b;
1968 }
1969 let result = dyn_alignr_128(
1970 self,
1971 self.cvt_to_bytes_u16x8(b).val.0,
1972 self.cvt_to_bytes_u16x8(a).val.0,
1973 SHIFT * 2usize,
1974 );
1975 self.cvt_from_bytes_u16x8(u8x16 {
1976 val: crate::support::Aligned128(result),
1977 simd: self,
1978 })
1979 }
1980 #[inline(always)]
1981 fn slide_within_blocks_u16x8<const SHIFT: usize>(
1982 self,
1983 a: u16x8<Self>,
1984 b: u16x8<Self>,
1985 ) -> u16x8<Self> {
1986 self.slide_u16x8::<SHIFT>(a, b)
1987 }
1988 #[inline(always)]
1989 fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8<Self>, indices: u8x16<Self>) -> u16x8<Self> {
1990 crate::kernel!(
1991 #[inline(always)]
1992 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, indices: u8x16<Sse4_2>) -> u16x8<Sse4_2> {
1993 let result = _mm_shuffle_epi8(token.cvt_to_bytes_u16x8(a).val.0, indices.into());
1994 token.cvt_from_bytes_u16x8(u8x16 {
1995 val: crate::support::Aligned128(result),
1996 simd: token,
1997 })
1998 }
1999 );
2000 kernel(self, a, indices)
2001 }
2002 #[inline(always)]
2003 fn add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2004 crate::kernel!(
2005 #[inline(always)]
2006 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
2007 _mm_add_epi16(a.into(), b.into()).simd_into(token)
2008 }
2009 );
2010 kernel(self, a, b)
2011 }
2012 #[inline(always)]
2013 fn sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2014 crate::kernel!(
2015 #[inline(always)]
2016 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
2017 _mm_sub_epi16(a.into(), b.into()).simd_into(token)
2018 }
2019 );
2020 kernel(self, a, b)
2021 }
2022 #[inline(always)]
2023 fn mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2024 crate::kernel!(
2025 #[inline(always)]
2026 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
2027 _mm_mullo_epi16(a.into(), b.into()).simd_into(token)
2028 }
2029 );
2030 kernel(self, a, b)
2031 }
2032 #[inline(always)]
2033 fn and_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2034 crate::kernel!(
2035 #[inline(always)]
2036 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
2037 _mm_and_si128(a.into(), b.into()).simd_into(token)
2038 }
2039 );
2040 kernel(self, a, b)
2041 }
2042 #[inline(always)]
2043 fn or_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2044 crate::kernel!(
2045 #[inline(always)]
2046 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
2047 _mm_or_si128(a.into(), b.into()).simd_into(token)
2048 }
2049 );
2050 kernel(self, a, b)
2051 }
2052 #[inline(always)]
2053 fn xor_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2054 crate::kernel!(
2055 #[inline(always)]
2056 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
2057 _mm_xor_si128(a.into(), b.into()).simd_into(token)
2058 }
2059 );
2060 kernel(self, a, b)
2061 }
2062 #[inline(always)]
2063 fn not_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
2064 a ^ !0
2065 }
2066 #[inline(always)]
2067 fn shl_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
2068 crate::kernel!(
2069 #[inline(always)]
2070 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, shift: u32) -> u16x8<Sse4_2> {
2071 _mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
2072 }
2073 );
2074 kernel(self, a, shift)
2075 }
2076 #[inline(always)]
2077 fn shlv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2078 core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self)
2079 }
2080 #[inline(always)]
2081 fn shr_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
2082 crate::kernel!(
2083 #[inline(always)]
2084 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, shift: u32) -> u16x8<Sse4_2> {
2085 _mm_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
2086 }
2087 );
2088 kernel(self, a, shift)
2089 }
2090 #[inline(always)]
2091 fn shrv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2092 core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self)
2093 }
2094 #[inline(always)]
2095 fn simd_eq_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
2096 crate::kernel!(
2097 #[inline(always)]
2098 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> mask16x8<Sse4_2> {
2099 _mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
2100 }
2101 );
2102 kernel(self, a, b)
2103 }
2104 #[inline(always)]
2105 fn simd_lt_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
2106 crate::kernel!(
2107 #[inline(always)]
2108 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> mask16x8<Sse4_2> {
2109 let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
2110 let a_signed = _mm_xor_si128(a.into(), sign_bit);
2111 let b_signed = _mm_xor_si128(b.into(), sign_bit);
2112 _mm_cmpgt_epi16(b_signed, a_signed).simd_into(token)
2113 }
2114 );
2115 kernel(self, a, b)
2116 }
2117 #[inline(always)]
2118 fn simd_le_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
2119 crate::kernel!(
2120 #[inline(always)]
2121 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> mask16x8<Sse4_2> {
2122 _mm_cmpeq_epi16(_mm_min_epu16(a.into(), b.into()), a.into()).simd_into(token)
2123 }
2124 );
2125 kernel(self, a, b)
2126 }
2127 #[inline(always)]
2128 fn simd_ge_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
2129 crate::kernel!(
2130 #[inline(always)]
2131 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> mask16x8<Sse4_2> {
2132 _mm_cmpeq_epi16(_mm_max_epu16(a.into(), b.into()), a.into()).simd_into(token)
2133 }
2134 );
2135 kernel(self, a, b)
2136 }
2137 #[inline(always)]
2138 fn simd_gt_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
2139 crate::kernel!(
2140 #[inline(always)]
2141 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> mask16x8<Sse4_2> {
2142 let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
2143 let a_signed = _mm_xor_si128(a.into(), sign_bit);
2144 let b_signed = _mm_xor_si128(b.into(), sign_bit);
2145 _mm_cmpgt_epi16(a_signed, b_signed).simd_into(token)
2146 }
2147 );
2148 kernel(self, a, b)
2149 }
2150 #[inline(always)]
2151 fn zip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2152 crate::kernel!(
2153 #[inline(always)]
2154 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
2155 _mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
2156 }
2157 );
2158 kernel(self, a, b)
2159 }
2160 #[inline(always)]
2161 fn zip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2162 crate::kernel!(
2163 #[inline(always)]
2164 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
2165 _mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
2166 }
2167 );
2168 kernel(self, a, b)
2169 }
2170 #[inline(always)]
2171 fn unzip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2172 crate::kernel!(
2173 #[inline(always)]
2174 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
2175 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
2176 let t1 = _mm_shuffle_epi8(a.into(), mask);
2177 let t2 = _mm_shuffle_epi8(b.into(), mask);
2178 _mm_unpacklo_epi64(t1, t2).simd_into(token)
2179 }
2180 );
2181 kernel(self, a, b)
2182 }
2183 #[inline(always)]
2184 fn unzip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2185 crate::kernel!(
2186 #[inline(always)]
2187 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
2188 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
2189 let t1 = _mm_shuffle_epi8(a.into(), mask);
2190 let t2 = _mm_shuffle_epi8(b.into(), mask);
2191 _mm_unpackhi_epi64(t1, t2).simd_into(token)
2192 }
2193 );
2194 kernel(self, a, b)
2195 }
2196 #[inline(always)]
2197 fn interleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
2198 (self.zip_low_u16x8(a, b), self.zip_high_u16x8(a, b))
2199 }
2200 #[inline(always)]
2201 fn deinterleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
2202 (self.unzip_low_u16x8(a, b), self.unzip_high_u16x8(a, b))
2203 }
2204 #[inline(always)]
2205 fn select_u16x8(self, a: mask16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
2206 crate::kernel!(
2207 #[inline(always)]
2208 fn kernel(
2209 token: Sse4_2,
2210 a: mask16x8<Sse4_2>,
2211 b: u16x8<Sse4_2>,
2212 c: u16x8<Sse4_2>,
2213 ) -> u16x8<Sse4_2> {
2214 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
2215 }
2216 );
2217 kernel(self, a, b, c)
2218 }
2219 #[inline(always)]
2220 fn min_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2221 crate::kernel!(
2222 #[inline(always)]
2223 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
2224 _mm_min_epu16(a.into(), b.into()).simd_into(token)
2225 }
2226 );
2227 kernel(self, a, b)
2228 }
2229 #[inline(always)]
2230 fn max_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
2231 crate::kernel!(
2232 #[inline(always)]
2233 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
2234 _mm_max_epu16(a.into(), b.into()).simd_into(token)
2235 }
2236 );
2237 kernel(self, a, b)
2238 }
2239 #[inline(always)]
2240 fn combine_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x16<Self> {
2241 u16x16 {
2242 val: crate::support::Aligned256([a.val.0, b.val.0]),
2243 simd: self,
2244 }
2245 }
2246 #[inline(always)]
2247 fn reinterpret_u8_u16x8(self, a: u16x8<Self>) -> u8x16<Self> {
2248 crate::kernel!(
2249 #[inline(always)]
2250 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>) -> u8x16<Sse4_2> {
2251 __m128i::from(a).simd_into(token)
2252 }
2253 );
2254 kernel(self, a)
2255 }
2256 #[inline(always)]
2257 fn reinterpret_u32_u16x8(self, a: u16x8<Self>) -> u32x4<Self> {
2258 crate::kernel!(
2259 #[inline(always)]
2260 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>) -> u32x4<Sse4_2> {
2261 __m128i::from(a).simd_into(token)
2262 }
2263 );
2264 kernel(self, a)
2265 }
2266 #[inline(always)]
2267 fn splat_mask16x8(self, val: bool) -> mask16x8<Self> {
2268 crate::kernel!(
2269 #[inline(always)]
2270 fn kernel(token: Sse4_2, val: bool) -> mask16x8<Sse4_2> {
2271 let val: i16 = if val { !0 } else { 0 };
2272 _mm_set1_epi16(val).simd_into(token)
2273 }
2274 );
2275 kernel(self, val)
2276 }
2277 #[inline(always)]
2278 fn load_array_mask16x8(self, val: [i16; 8usize]) -> mask16x8<Self> {
2279 mask16x8 {
2280 val: crate::transmute::checked_transmute_copy(&val),
2281 simd: self,
2282 }
2283 }
2284 #[inline(always)]
2285 fn as_array_mask16x8(self, a: mask16x8<Self>) -> [i16; 8usize] {
2286 crate::transmute::checked_transmute_copy::<__m128i, [i16; 8usize]>(&a.val.0)
2287 }
2288 #[inline(always)]
2289 fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8<Self> {
2290 crate::kernel!(
2291 #[inline(always)]
2292 fn kernel(token: Sse4_2, bits: u64) -> mask16x8<Sse4_2> {
2293 {
2294 let bit_lanes = _mm_set1_epi16(bits as i16);
2295 let bit_mask = _mm_setr_epi16(1, 2, 4, 8, 16, 32, 64, 128);
2296 _mm_cmpeq_epi16(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
2297 }
2298 .simd_into(token)
2299 }
2300 );
2301 kernel(self, bits)
2302 }
2303 #[inline(always)]
2304 fn to_bitmask_mask16x8(self, a: mask16x8<Self>) -> u64 {
2305 crate::kernel!(
2306 #[inline(always)]
2307 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>) -> u64 {
2308 {
2309 let packed = _mm_packs_epi16(a.into(), a.into());
2310 _mm_movemask_epi8(packed) as u8 as u64
2311 }
2312 }
2313 );
2314 kernel(self, a)
2315 }
2316 #[inline(always)]
2317 fn set_mask16x8(self, a: &mut mask16x8<Self>, index: usize, value: bool) -> () {
2318 assert!(
2319 index < 8usize,
2320 "mask lane index {index} is out of bounds for {} lanes",
2321 8usize
2322 );
2323 let mut lanes = self.as_array_mask16x8(*a);
2324 lanes[index] = if value { !0 } else { 0 };
2325 *a = self.load_array_mask16x8(lanes);
2326 }
2327 #[inline(always)]
2328 fn and_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
2329 crate::kernel!(
2330 #[inline(always)]
2331 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>, b: mask16x8<Sse4_2>) -> mask16x8<Sse4_2> {
2332 _mm_and_si128(a.into(), b.into()).simd_into(token)
2333 }
2334 );
2335 kernel(self, a, b)
2336 }
2337 #[inline(always)]
2338 fn or_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
2339 crate::kernel!(
2340 #[inline(always)]
2341 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>, b: mask16x8<Sse4_2>) -> mask16x8<Sse4_2> {
2342 _mm_or_si128(a.into(), b.into()).simd_into(token)
2343 }
2344 );
2345 kernel(self, a, b)
2346 }
2347 #[inline(always)]
2348 fn xor_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
2349 crate::kernel!(
2350 #[inline(always)]
2351 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>, b: mask16x8<Sse4_2>) -> mask16x8<Sse4_2> {
2352 _mm_xor_si128(a.into(), b.into()).simd_into(token)
2353 }
2354 );
2355 kernel(self, a, b)
2356 }
2357 #[inline(always)]
2358 fn not_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
2359 self.xor_mask16x8(a, self.splat_mask16x8(true))
2360 }
2361 #[inline(always)]
2362 fn select_mask16x8(
2363 self,
2364 a: mask16x8<Self>,
2365 b: mask16x8<Self>,
2366 c: mask16x8<Self>,
2367 ) -> mask16x8<Self> {
2368 crate::kernel!(
2369 #[inline(always)]
2370 fn kernel(
2371 token: Sse4_2,
2372 a: mask16x8<Sse4_2>,
2373 b: mask16x8<Sse4_2>,
2374 c: mask16x8<Sse4_2>,
2375 ) -> mask16x8<Sse4_2> {
2376 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
2377 }
2378 );
2379 kernel(self, a, b, c)
2380 }
2381 #[inline(always)]
2382 fn simd_eq_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
2383 crate::kernel!(
2384 #[inline(always)]
2385 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>, b: mask16x8<Sse4_2>) -> mask16x8<Sse4_2> {
2386 _mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
2387 }
2388 );
2389 kernel(self, a, b)
2390 }
2391 #[inline(always)]
2392 fn any_true_mask16x8(self, a: mask16x8<Self>) -> bool {
2393 crate::kernel!(
2394 #[inline(always)]
2395 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>) -> bool {
2396 _mm_movemask_epi8(a.into()) as u32 != 0
2397 }
2398 );
2399 kernel(self, a)
2400 }
2401 #[inline(always)]
2402 fn all_true_mask16x8(self, a: mask16x8<Self>) -> bool {
2403 crate::kernel!(
2404 #[inline(always)]
2405 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>) -> bool {
2406 _mm_movemask_epi8(a.into()) as u32 == 0xffff
2407 }
2408 );
2409 kernel(self, a)
2410 }
2411 #[inline(always)]
2412 fn any_false_mask16x8(self, a: mask16x8<Self>) -> bool {
2413 crate::kernel!(
2414 #[inline(always)]
2415 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>) -> bool {
2416 _mm_movemask_epi8(a.into()) as u32 != 0xffff
2417 }
2418 );
2419 kernel(self, a)
2420 }
2421 #[inline(always)]
2422 fn all_false_mask16x8(self, a: mask16x8<Self>) -> bool {
2423 crate::kernel!(
2424 #[inline(always)]
2425 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>) -> bool {
2426 _mm_movemask_epi8(a.into()) as u32 == 0
2427 }
2428 );
2429 kernel(self, a)
2430 }
2431 #[inline(always)]
2432 fn combine_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x16<Self> {
2433 mask16x16 {
2434 val: crate::support::Aligned256([a.val.0, b.val.0]),
2435 simd: self,
2436 }
2437 }
2438 #[inline(always)]
2439 fn splat_i32x4(self, val: i32) -> i32x4<Self> {
2440 crate::kernel!(
2441 #[inline(always)]
2442 fn kernel(token: Sse4_2, val: i32) -> i32x4<Sse4_2> {
2443 _mm_set1_epi32(val).simd_into(token)
2444 }
2445 );
2446 kernel(self, val)
2447 }
2448 #[inline(always)]
2449 fn load_array_i32x4(self, val: [i32; 4usize]) -> i32x4<Self> {
2450 i32x4 {
2451 val: crate::transmute::checked_transmute_copy(&val),
2452 simd: self,
2453 }
2454 }
2455 #[inline(always)]
2456 fn load_array_ref_i32x4(self, val: &[i32; 4usize]) -> i32x4<Self> {
2457 i32x4 {
2458 val: crate::transmute::checked_transmute_copy(val),
2459 simd: self,
2460 }
2461 }
2462 #[inline(always)]
2463 fn as_array_i32x4(self, a: i32x4<Self>) -> [i32; 4usize] {
2464 crate::transmute::checked_transmute_copy::<__m128i, [i32; 4usize]>(&a.val.0)
2465 }
2466 #[inline(always)]
2467 fn as_array_ref_i32x4(self, a: &i32x4<Self>) -> &[i32; 4usize] {
2468 crate::transmute::checked_cast_ref::<__m128i, [i32; 4usize]>(&a.val.0)
2469 }
2470 #[inline(always)]
2471 fn as_array_mut_i32x4(self, a: &mut i32x4<Self>) -> &mut [i32; 4usize] {
2472 crate::transmute::checked_cast_mut::<__m128i, [i32; 4usize]>(&mut a.val.0)
2473 }
2474 #[inline(always)]
2475 fn store_array_i32x4(self, a: i32x4<Self>, dest: &mut [i32; 4usize]) -> () {
2476 crate::transmute::checked_transmute_store(a.val.0, dest);
2477 }
2478 #[inline(always)]
2479 fn cvt_from_bytes_i32x4(self, a: u8x16<Self>) -> i32x4<Self> {
2480 i32x4 {
2481 val: crate::transmute::checked_transmute_copy(&a.val),
2482 simd: self,
2483 }
2484 }
2485 #[inline(always)]
2486 fn cvt_to_bytes_i32x4(self, a: i32x4<Self>) -> u8x16<Self> {
2487 u8x16 {
2488 val: crate::transmute::checked_transmute_copy(&a.val),
2489 simd: self,
2490 }
2491 }
2492 #[inline(always)]
2493 fn slide_i32x4<const SHIFT: usize>(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2494 if SHIFT >= 4usize {
2495 return b;
2496 }
2497 let result = dyn_alignr_128(
2498 self,
2499 self.cvt_to_bytes_i32x4(b).val.0,
2500 self.cvt_to_bytes_i32x4(a).val.0,
2501 SHIFT * 4usize,
2502 );
2503 self.cvt_from_bytes_i32x4(u8x16 {
2504 val: crate::support::Aligned128(result),
2505 simd: self,
2506 })
2507 }
2508 #[inline(always)]
2509 fn slide_within_blocks_i32x4<const SHIFT: usize>(
2510 self,
2511 a: i32x4<Self>,
2512 b: i32x4<Self>,
2513 ) -> i32x4<Self> {
2514 self.slide_i32x4::<SHIFT>(a, b)
2515 }
2516 #[inline(always)]
2517 fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4<Self>, indices: u8x16<Self>) -> i32x4<Self> {
2518 crate::kernel!(
2519 #[inline(always)]
2520 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, indices: u8x16<Sse4_2>) -> i32x4<Sse4_2> {
2521 let result = _mm_shuffle_epi8(token.cvt_to_bytes_i32x4(a).val.0, indices.into());
2522 token.cvt_from_bytes_i32x4(u8x16 {
2523 val: crate::support::Aligned128(result),
2524 simd: token,
2525 })
2526 }
2527 );
2528 kernel(self, a, indices)
2529 }
2530 #[inline(always)]
2531 fn add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2532 crate::kernel!(
2533 #[inline(always)]
2534 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
2535 _mm_add_epi32(a.into(), b.into()).simd_into(token)
2536 }
2537 );
2538 kernel(self, a, b)
2539 }
2540 #[inline(always)]
2541 fn sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2542 crate::kernel!(
2543 #[inline(always)]
2544 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
2545 _mm_sub_epi32(a.into(), b.into()).simd_into(token)
2546 }
2547 );
2548 kernel(self, a, b)
2549 }
2550 #[inline(always)]
2551 fn mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2552 crate::kernel!(
2553 #[inline(always)]
2554 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
2555 _mm_mullo_epi32(a.into(), b.into()).simd_into(token)
2556 }
2557 );
2558 kernel(self, a, b)
2559 }
2560 #[inline(always)]
2561 fn and_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2562 crate::kernel!(
2563 #[inline(always)]
2564 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
2565 _mm_and_si128(a.into(), b.into()).simd_into(token)
2566 }
2567 );
2568 kernel(self, a, b)
2569 }
2570 #[inline(always)]
2571 fn or_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2572 crate::kernel!(
2573 #[inline(always)]
2574 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
2575 _mm_or_si128(a.into(), b.into()).simd_into(token)
2576 }
2577 );
2578 kernel(self, a, b)
2579 }
2580 #[inline(always)]
2581 fn xor_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2582 crate::kernel!(
2583 #[inline(always)]
2584 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
2585 _mm_xor_si128(a.into(), b.into()).simd_into(token)
2586 }
2587 );
2588 kernel(self, a, b)
2589 }
2590 #[inline(always)]
2591 fn not_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
2592 a ^ !0
2593 }
2594 #[inline(always)]
2595 fn shl_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
2596 crate::kernel!(
2597 #[inline(always)]
2598 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, shift: u32) -> i32x4<Sse4_2> {
2599 _mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
2600 }
2601 );
2602 kernel(self, a, shift)
2603 }
2604 #[inline(always)]
2605 fn shlv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2606 core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self)
2607 }
2608 #[inline(always)]
2609 fn shr_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
2610 crate::kernel!(
2611 #[inline(always)]
2612 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, shift: u32) -> i32x4<Sse4_2> {
2613 _mm_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
2614 }
2615 );
2616 kernel(self, a, shift)
2617 }
2618 #[inline(always)]
2619 fn shrv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2620 core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self)
2621 }
2622 #[inline(always)]
2623 fn simd_eq_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
2624 crate::kernel!(
2625 #[inline(always)]
2626 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> mask32x4<Sse4_2> {
2627 _mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
2628 }
2629 );
2630 kernel(self, a, b)
2631 }
2632 #[inline(always)]
2633 fn simd_lt_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
2634 crate::kernel!(
2635 #[inline(always)]
2636 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> mask32x4<Sse4_2> {
2637 _mm_cmpgt_epi32(b.into(), a.into()).simd_into(token)
2638 }
2639 );
2640 kernel(self, a, b)
2641 }
2642 #[inline(always)]
2643 fn simd_le_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
2644 crate::kernel!(
2645 #[inline(always)]
2646 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> mask32x4<Sse4_2> {
2647 _mm_cmpeq_epi32(_mm_min_epi32(a.into(), b.into()), a.into()).simd_into(token)
2648 }
2649 );
2650 kernel(self, a, b)
2651 }
2652 #[inline(always)]
2653 fn simd_ge_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
2654 crate::kernel!(
2655 #[inline(always)]
2656 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> mask32x4<Sse4_2> {
2657 _mm_cmpeq_epi32(_mm_max_epi32(a.into(), b.into()), a.into()).simd_into(token)
2658 }
2659 );
2660 kernel(self, a, b)
2661 }
2662 #[inline(always)]
2663 fn simd_gt_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
2664 crate::kernel!(
2665 #[inline(always)]
2666 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> mask32x4<Sse4_2> {
2667 _mm_cmpgt_epi32(a.into(), b.into()).simd_into(token)
2668 }
2669 );
2670 kernel(self, a, b)
2671 }
2672 #[inline(always)]
2673 fn zip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2674 crate::kernel!(
2675 #[inline(always)]
2676 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
2677 _mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
2678 }
2679 );
2680 kernel(self, a, b)
2681 }
2682 #[inline(always)]
2683 fn zip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2684 crate::kernel!(
2685 #[inline(always)]
2686 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
2687 _mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
2688 }
2689 );
2690 kernel(self, a, b)
2691 }
2692 #[inline(always)]
2693 fn unzip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2694 crate::kernel!(
2695 #[inline(always)]
2696 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
2697 let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
2698 let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
2699 _mm_unpacklo_epi64(t1, t2).simd_into(token)
2700 }
2701 );
2702 kernel(self, a, b)
2703 }
2704 #[inline(always)]
2705 fn unzip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2706 crate::kernel!(
2707 #[inline(always)]
2708 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
2709 let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
2710 let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
2711 _mm_unpackhi_epi64(t1, t2).simd_into(token)
2712 }
2713 );
2714 kernel(self, a, b)
2715 }
2716 #[inline(always)]
2717 fn interleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
2718 (self.zip_low_i32x4(a, b), self.zip_high_i32x4(a, b))
2719 }
2720 #[inline(always)]
2721 fn deinterleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
2722 (self.unzip_low_i32x4(a, b), self.unzip_high_i32x4(a, b))
2723 }
2724 #[inline(always)]
2725 fn select_i32x4(self, a: mask32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
2726 crate::kernel!(
2727 #[inline(always)]
2728 fn kernel(
2729 token: Sse4_2,
2730 a: mask32x4<Sse4_2>,
2731 b: i32x4<Sse4_2>,
2732 c: i32x4<Sse4_2>,
2733 ) -> i32x4<Sse4_2> {
2734 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
2735 }
2736 );
2737 kernel(self, a, b, c)
2738 }
2739 #[inline(always)]
2740 fn min_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2741 crate::kernel!(
2742 #[inline(always)]
2743 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
2744 _mm_min_epi32(a.into(), b.into()).simd_into(token)
2745 }
2746 );
2747 kernel(self, a, b)
2748 }
2749 #[inline(always)]
2750 fn max_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
2751 crate::kernel!(
2752 #[inline(always)]
2753 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
2754 _mm_max_epi32(a.into(), b.into()).simd_into(token)
2755 }
2756 );
2757 kernel(self, a, b)
2758 }
2759 #[inline(always)]
2760 fn combine_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x8<Self> {
2761 i32x8 {
2762 val: crate::support::Aligned256([a.val.0, b.val.0]),
2763 simd: self,
2764 }
2765 }
2766 #[inline(always)]
2767 fn neg_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
2768 crate::kernel!(
2769 #[inline(always)]
2770 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
2771 _mm_sub_epi32(_mm_setzero_si128(), a.into()).simd_into(token)
2772 }
2773 );
2774 kernel(self, a)
2775 }
2776 #[inline(always)]
2777 fn reinterpret_u8_i32x4(self, a: i32x4<Self>) -> u8x16<Self> {
2778 crate::kernel!(
2779 #[inline(always)]
2780 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>) -> u8x16<Sse4_2> {
2781 __m128i::from(a).simd_into(token)
2782 }
2783 );
2784 kernel(self, a)
2785 }
2786 #[inline(always)]
2787 fn reinterpret_u32_i32x4(self, a: i32x4<Self>) -> u32x4<Self> {
2788 crate::kernel!(
2789 #[inline(always)]
2790 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>) -> u32x4<Sse4_2> {
2791 __m128i::from(a).simd_into(token)
2792 }
2793 );
2794 kernel(self, a)
2795 }
2796 #[inline(always)]
2797 fn cvt_f32_i32x4(self, a: i32x4<Self>) -> f32x4<Self> {
2798 crate::kernel!(
2799 #[inline(always)]
2800 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>) -> f32x4<Sse4_2> {
2801 _mm_cvtepi32_ps(a.into()).simd_into(token)
2802 }
2803 );
2804 kernel(self, a)
2805 }
2806 #[inline(always)]
2807 fn splat_u32x4(self, val: u32) -> u32x4<Self> {
2808 crate::kernel!(
2809 #[inline(always)]
2810 fn kernel(token: Sse4_2, val: u32) -> u32x4<Sse4_2> {
2811 _mm_set1_epi32(val.cast_signed()).simd_into(token)
2812 }
2813 );
2814 kernel(self, val)
2815 }
2816 #[inline(always)]
2817 fn load_array_u32x4(self, val: [u32; 4usize]) -> u32x4<Self> {
2818 u32x4 {
2819 val: crate::transmute::checked_transmute_copy(&val),
2820 simd: self,
2821 }
2822 }
2823 #[inline(always)]
2824 fn load_array_ref_u32x4(self, val: &[u32; 4usize]) -> u32x4<Self> {
2825 u32x4 {
2826 val: crate::transmute::checked_transmute_copy(val),
2827 simd: self,
2828 }
2829 }
2830 #[inline(always)]
2831 fn as_array_u32x4(self, a: u32x4<Self>) -> [u32; 4usize] {
2832 crate::transmute::checked_transmute_copy::<__m128i, [u32; 4usize]>(&a.val.0)
2833 }
2834 #[inline(always)]
2835 fn as_array_ref_u32x4(self, a: &u32x4<Self>) -> &[u32; 4usize] {
2836 crate::transmute::checked_cast_ref::<__m128i, [u32; 4usize]>(&a.val.0)
2837 }
2838 #[inline(always)]
2839 fn as_array_mut_u32x4(self, a: &mut u32x4<Self>) -> &mut [u32; 4usize] {
2840 crate::transmute::checked_cast_mut::<__m128i, [u32; 4usize]>(&mut a.val.0)
2841 }
2842 #[inline(always)]
2843 fn store_array_u32x4(self, a: u32x4<Self>, dest: &mut [u32; 4usize]) -> () {
2844 crate::transmute::checked_transmute_store(a.val.0, dest);
2845 }
2846 #[inline(always)]
2847 fn cvt_from_bytes_u32x4(self, a: u8x16<Self>) -> u32x4<Self> {
2848 u32x4 {
2849 val: crate::transmute::checked_transmute_copy(&a.val),
2850 simd: self,
2851 }
2852 }
2853 #[inline(always)]
2854 fn cvt_to_bytes_u32x4(self, a: u32x4<Self>) -> u8x16<Self> {
2855 u8x16 {
2856 val: crate::transmute::checked_transmute_copy(&a.val),
2857 simd: self,
2858 }
2859 }
2860 #[inline(always)]
2861 fn slide_u32x4<const SHIFT: usize>(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
2862 if SHIFT >= 4usize {
2863 return b;
2864 }
2865 let result = dyn_alignr_128(
2866 self,
2867 self.cvt_to_bytes_u32x4(b).val.0,
2868 self.cvt_to_bytes_u32x4(a).val.0,
2869 SHIFT * 4usize,
2870 );
2871 self.cvt_from_bytes_u32x4(u8x16 {
2872 val: crate::support::Aligned128(result),
2873 simd: self,
2874 })
2875 }
2876 #[inline(always)]
2877 fn slide_within_blocks_u32x4<const SHIFT: usize>(
2878 self,
2879 a: u32x4<Self>,
2880 b: u32x4<Self>,
2881 ) -> u32x4<Self> {
2882 self.slide_u32x4::<SHIFT>(a, b)
2883 }
2884 #[inline(always)]
2885 fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4<Self>, indices: u8x16<Self>) -> u32x4<Self> {
2886 crate::kernel!(
2887 #[inline(always)]
2888 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, indices: u8x16<Sse4_2>) -> u32x4<Sse4_2> {
2889 let result = _mm_shuffle_epi8(token.cvt_to_bytes_u32x4(a).val.0, indices.into());
2890 token.cvt_from_bytes_u32x4(u8x16 {
2891 val: crate::support::Aligned128(result),
2892 simd: token,
2893 })
2894 }
2895 );
2896 kernel(self, a, indices)
2897 }
2898 #[inline(always)]
2899 fn add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
2900 crate::kernel!(
2901 #[inline(always)]
2902 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
2903 _mm_add_epi32(a.into(), b.into()).simd_into(token)
2904 }
2905 );
2906 kernel(self, a, b)
2907 }
2908 #[inline(always)]
2909 fn sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
2910 crate::kernel!(
2911 #[inline(always)]
2912 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
2913 _mm_sub_epi32(a.into(), b.into()).simd_into(token)
2914 }
2915 );
2916 kernel(self, a, b)
2917 }
2918 #[inline(always)]
2919 fn mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
2920 crate::kernel!(
2921 #[inline(always)]
2922 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
2923 _mm_mullo_epi32(a.into(), b.into()).simd_into(token)
2924 }
2925 );
2926 kernel(self, a, b)
2927 }
2928 #[inline(always)]
2929 fn and_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
2930 crate::kernel!(
2931 #[inline(always)]
2932 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
2933 _mm_and_si128(a.into(), b.into()).simd_into(token)
2934 }
2935 );
2936 kernel(self, a, b)
2937 }
2938 #[inline(always)]
2939 fn or_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
2940 crate::kernel!(
2941 #[inline(always)]
2942 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
2943 _mm_or_si128(a.into(), b.into()).simd_into(token)
2944 }
2945 );
2946 kernel(self, a, b)
2947 }
2948 #[inline(always)]
2949 fn xor_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
2950 crate::kernel!(
2951 #[inline(always)]
2952 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
2953 _mm_xor_si128(a.into(), b.into()).simd_into(token)
2954 }
2955 );
2956 kernel(self, a, b)
2957 }
2958 #[inline(always)]
2959 fn not_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
2960 a ^ !0
2961 }
2962 #[inline(always)]
2963 fn shl_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
2964 crate::kernel!(
2965 #[inline(always)]
2966 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, shift: u32) -> u32x4<Sse4_2> {
2967 _mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
2968 }
2969 );
2970 kernel(self, a, shift)
2971 }
2972 #[inline(always)]
2973 fn shlv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
2974 core::array::from_fn(|i| core::ops::Shl::shl(a[i], b[i])).simd_into(self)
2975 }
2976 #[inline(always)]
2977 fn shr_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
2978 crate::kernel!(
2979 #[inline(always)]
2980 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, shift: u32) -> u32x4<Sse4_2> {
2981 _mm_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
2982 }
2983 );
2984 kernel(self, a, shift)
2985 }
2986 #[inline(always)]
2987 fn shrv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
2988 core::array::from_fn(|i| core::ops::Shr::shr(a[i], b[i])).simd_into(self)
2989 }
2990 #[inline(always)]
2991 fn simd_eq_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
2992 crate::kernel!(
2993 #[inline(always)]
2994 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> mask32x4<Sse4_2> {
2995 _mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
2996 }
2997 );
2998 kernel(self, a, b)
2999 }
3000 #[inline(always)]
3001 fn simd_lt_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
3002 crate::kernel!(
3003 #[inline(always)]
3004 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> mask32x4<Sse4_2> {
3005 let sign_bit = _mm_set1_epi32(0x80000000u32.cast_signed());
3006 let a_signed = _mm_xor_si128(a.into(), sign_bit);
3007 let b_signed = _mm_xor_si128(b.into(), sign_bit);
3008 _mm_cmpgt_epi32(b_signed, a_signed).simd_into(token)
3009 }
3010 );
3011 kernel(self, a, b)
3012 }
3013 #[inline(always)]
3014 fn simd_le_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
3015 crate::kernel!(
3016 #[inline(always)]
3017 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> mask32x4<Sse4_2> {
3018 _mm_cmpeq_epi32(_mm_min_epu32(a.into(), b.into()), a.into()).simd_into(token)
3019 }
3020 );
3021 kernel(self, a, b)
3022 }
3023 #[inline(always)]
3024 fn simd_ge_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
3025 crate::kernel!(
3026 #[inline(always)]
3027 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> mask32x4<Sse4_2> {
3028 _mm_cmpeq_epi32(_mm_max_epu32(a.into(), b.into()), a.into()).simd_into(token)
3029 }
3030 );
3031 kernel(self, a, b)
3032 }
3033 #[inline(always)]
3034 fn simd_gt_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
3035 crate::kernel!(
3036 #[inline(always)]
3037 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> mask32x4<Sse4_2> {
3038 let sign_bit = _mm_set1_epi32(0x80000000u32.cast_signed());
3039 let a_signed = _mm_xor_si128(a.into(), sign_bit);
3040 let b_signed = _mm_xor_si128(b.into(), sign_bit);
3041 _mm_cmpgt_epi32(a_signed, b_signed).simd_into(token)
3042 }
3043 );
3044 kernel(self, a, b)
3045 }
3046 #[inline(always)]
3047 fn zip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
3048 crate::kernel!(
3049 #[inline(always)]
3050 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
3051 _mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
3052 }
3053 );
3054 kernel(self, a, b)
3055 }
3056 #[inline(always)]
3057 fn zip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
3058 crate::kernel!(
3059 #[inline(always)]
3060 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
3061 _mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
3062 }
3063 );
3064 kernel(self, a, b)
3065 }
3066 #[inline(always)]
3067 fn unzip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
3068 crate::kernel!(
3069 #[inline(always)]
3070 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
3071 let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
3072 let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
3073 _mm_unpacklo_epi64(t1, t2).simd_into(token)
3074 }
3075 );
3076 kernel(self, a, b)
3077 }
3078 #[inline(always)]
3079 fn unzip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
3080 crate::kernel!(
3081 #[inline(always)]
3082 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
3083 let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
3084 let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
3085 _mm_unpackhi_epi64(t1, t2).simd_into(token)
3086 }
3087 );
3088 kernel(self, a, b)
3089 }
3090 #[inline(always)]
3091 fn interleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
3092 (self.zip_low_u32x4(a, b), self.zip_high_u32x4(a, b))
3093 }
3094 #[inline(always)]
3095 fn deinterleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
3096 (self.unzip_low_u32x4(a, b), self.unzip_high_u32x4(a, b))
3097 }
3098 #[inline(always)]
3099 fn select_u32x4(self, a: mask32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
3100 crate::kernel!(
3101 #[inline(always)]
3102 fn kernel(
3103 token: Sse4_2,
3104 a: mask32x4<Sse4_2>,
3105 b: u32x4<Sse4_2>,
3106 c: u32x4<Sse4_2>,
3107 ) -> u32x4<Sse4_2> {
3108 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
3109 }
3110 );
3111 kernel(self, a, b, c)
3112 }
3113 #[inline(always)]
3114 fn min_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
3115 crate::kernel!(
3116 #[inline(always)]
3117 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
3118 _mm_min_epu32(a.into(), b.into()).simd_into(token)
3119 }
3120 );
3121 kernel(self, a, b)
3122 }
3123 #[inline(always)]
3124 fn max_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
3125 crate::kernel!(
3126 #[inline(always)]
3127 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
3128 _mm_max_epu32(a.into(), b.into()).simd_into(token)
3129 }
3130 );
3131 kernel(self, a, b)
3132 }
3133 #[inline(always)]
3134 fn combine_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x8<Self> {
3135 u32x8 {
3136 val: crate::support::Aligned256([a.val.0, b.val.0]),
3137 simd: self,
3138 }
3139 }
3140 #[inline(always)]
3141 fn reinterpret_u8_u32x4(self, a: u32x4<Self>) -> u8x16<Self> {
3142 crate::kernel!(
3143 #[inline(always)]
3144 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>) -> u8x16<Sse4_2> {
3145 __m128i::from(a).simd_into(token)
3146 }
3147 );
3148 kernel(self, a)
3149 }
3150 #[inline(always)]
3151 fn cvt_f32_u32x4(self, a: u32x4<Self>) -> f32x4<Self> {
3152 crate::kernel!(
3153 #[inline(always)]
3154 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>) -> f32x4<Sse4_2> {
3155 let a = a.into();
3156 let lo = _mm_blend_epi16::<0xAA>(a, _mm_set1_epi32(0x4B000000));
3157 let hi =
3158 _mm_blend_epi16::<0xAA>(_mm_srli_epi32::<16>(a), _mm_set1_epi32(0x53000000));
3159 let fhi = _mm_sub_ps(
3160 _mm_castsi128_ps(hi),
3161 _mm_set1_ps(f32::from_bits(0x53000080)),
3162 );
3163 let result = _mm_add_ps(_mm_castsi128_ps(lo), fhi);
3164 result.simd_into(token)
3165 }
3166 );
3167 kernel(self, a)
3168 }
3169 #[inline(always)]
3170 fn splat_mask32x4(self, val: bool) -> mask32x4<Self> {
3171 crate::kernel!(
3172 #[inline(always)]
3173 fn kernel(token: Sse4_2, val: bool) -> mask32x4<Sse4_2> {
3174 let val: i32 = if val { !0 } else { 0 };
3175 _mm_set1_epi32(val).simd_into(token)
3176 }
3177 );
3178 kernel(self, val)
3179 }
3180 #[inline(always)]
3181 fn load_array_mask32x4(self, val: [i32; 4usize]) -> mask32x4<Self> {
3182 mask32x4 {
3183 val: crate::transmute::checked_transmute_copy(&val),
3184 simd: self,
3185 }
3186 }
3187 #[inline(always)]
3188 fn as_array_mask32x4(self, a: mask32x4<Self>) -> [i32; 4usize] {
3189 crate::transmute::checked_transmute_copy::<__m128i, [i32; 4usize]>(&a.val.0)
3190 }
3191 #[inline(always)]
3192 fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4<Self> {
3193 crate::kernel!(
3194 #[inline(always)]
3195 fn kernel(token: Sse4_2, bits: u64) -> mask32x4<Sse4_2> {
3196 {
3197 let bit_lanes = _mm_set1_epi32(bits as i32);
3198 let bit_mask = _mm_setr_epi32(1, 2, 4, 8);
3199 _mm_cmpeq_epi32(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
3200 }
3201 .simd_into(token)
3202 }
3203 );
3204 kernel(self, bits)
3205 }
3206 #[inline(always)]
3207 fn to_bitmask_mask32x4(self, a: mask32x4<Self>) -> u64 {
3208 crate::kernel!(
3209 #[inline(always)]
3210 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>) -> u64 {
3211 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 as u64
3212 }
3213 );
3214 kernel(self, a)
3215 }
3216 #[inline(always)]
3217 fn set_mask32x4(self, a: &mut mask32x4<Self>, index: usize, value: bool) -> () {
3218 assert!(
3219 index < 4usize,
3220 "mask lane index {index} is out of bounds for {} lanes",
3221 4usize
3222 );
3223 let mut lanes = self.as_array_mask32x4(*a);
3224 lanes[index] = if value { !0 } else { 0 };
3225 *a = self.load_array_mask32x4(lanes);
3226 }
3227 #[inline(always)]
3228 fn and_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
3229 crate::kernel!(
3230 #[inline(always)]
3231 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>, b: mask32x4<Sse4_2>) -> mask32x4<Sse4_2> {
3232 _mm_and_si128(a.into(), b.into()).simd_into(token)
3233 }
3234 );
3235 kernel(self, a, b)
3236 }
3237 #[inline(always)]
3238 fn or_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
3239 crate::kernel!(
3240 #[inline(always)]
3241 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>, b: mask32x4<Sse4_2>) -> mask32x4<Sse4_2> {
3242 _mm_or_si128(a.into(), b.into()).simd_into(token)
3243 }
3244 );
3245 kernel(self, a, b)
3246 }
3247 #[inline(always)]
3248 fn xor_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
3249 crate::kernel!(
3250 #[inline(always)]
3251 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>, b: mask32x4<Sse4_2>) -> mask32x4<Sse4_2> {
3252 _mm_xor_si128(a.into(), b.into()).simd_into(token)
3253 }
3254 );
3255 kernel(self, a, b)
3256 }
3257 #[inline(always)]
3258 fn not_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
3259 self.xor_mask32x4(a, self.splat_mask32x4(true))
3260 }
3261 #[inline(always)]
3262 fn select_mask32x4(
3263 self,
3264 a: mask32x4<Self>,
3265 b: mask32x4<Self>,
3266 c: mask32x4<Self>,
3267 ) -> mask32x4<Self> {
3268 crate::kernel!(
3269 #[inline(always)]
3270 fn kernel(
3271 token: Sse4_2,
3272 a: mask32x4<Sse4_2>,
3273 b: mask32x4<Sse4_2>,
3274 c: mask32x4<Sse4_2>,
3275 ) -> mask32x4<Sse4_2> {
3276 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
3277 }
3278 );
3279 kernel(self, a, b, c)
3280 }
3281 #[inline(always)]
3282 fn simd_eq_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
3283 crate::kernel!(
3284 #[inline(always)]
3285 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>, b: mask32x4<Sse4_2>) -> mask32x4<Sse4_2> {
3286 _mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
3287 }
3288 );
3289 kernel(self, a, b)
3290 }
3291 #[inline(always)]
3292 fn any_true_mask32x4(self, a: mask32x4<Self>) -> bool {
3293 crate::kernel!(
3294 #[inline(always)]
3295 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>) -> bool {
3296 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 != 0
3297 }
3298 );
3299 kernel(self, a)
3300 }
3301 #[inline(always)]
3302 fn all_true_mask32x4(self, a: mask32x4<Self>) -> bool {
3303 crate::kernel!(
3304 #[inline(always)]
3305 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>) -> bool {
3306 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 == 0b1111
3307 }
3308 );
3309 kernel(self, a)
3310 }
3311 #[inline(always)]
3312 fn any_false_mask32x4(self, a: mask32x4<Self>) -> bool {
3313 crate::kernel!(
3314 #[inline(always)]
3315 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>) -> bool {
3316 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 != 0b1111
3317 }
3318 );
3319 kernel(self, a)
3320 }
3321 #[inline(always)]
3322 fn all_false_mask32x4(self, a: mask32x4<Self>) -> bool {
3323 crate::kernel!(
3324 #[inline(always)]
3325 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>) -> bool {
3326 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 == 0
3327 }
3328 );
3329 kernel(self, a)
3330 }
3331 #[inline(always)]
3332 fn combine_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x8<Self> {
3333 mask32x8 {
3334 val: crate::support::Aligned256([a.val.0, b.val.0]),
3335 simd: self,
3336 }
3337 }
3338 #[inline(always)]
3339 fn splat_f64x2(self, val: f64) -> f64x2<Self> {
3340 crate::kernel!(
3341 #[inline(always)]
3342 fn kernel(token: Sse4_2, val: f64) -> f64x2<Sse4_2> {
3343 _mm_set1_pd(val).simd_into(token)
3344 }
3345 );
3346 kernel(self, val)
3347 }
3348 #[inline(always)]
3349 fn load_array_f64x2(self, val: [f64; 2usize]) -> f64x2<Self> {
3350 f64x2 {
3351 val: crate::transmute::checked_transmute_copy(&val),
3352 simd: self,
3353 }
3354 }
3355 #[inline(always)]
3356 fn load_array_ref_f64x2(self, val: &[f64; 2usize]) -> f64x2<Self> {
3357 f64x2 {
3358 val: crate::transmute::checked_transmute_copy(val),
3359 simd: self,
3360 }
3361 }
3362 #[inline(always)]
3363 fn as_array_f64x2(self, a: f64x2<Self>) -> [f64; 2usize] {
3364 crate::transmute::checked_transmute_copy::<__m128d, [f64; 2usize]>(&a.val.0)
3365 }
3366 #[inline(always)]
3367 fn as_array_ref_f64x2(self, a: &f64x2<Self>) -> &[f64; 2usize] {
3368 crate::transmute::checked_cast_ref::<__m128d, [f64; 2usize]>(&a.val.0)
3369 }
3370 #[inline(always)]
3371 fn as_array_mut_f64x2(self, a: &mut f64x2<Self>) -> &mut [f64; 2usize] {
3372 crate::transmute::checked_cast_mut::<__m128d, [f64; 2usize]>(&mut a.val.0)
3373 }
3374 #[inline(always)]
3375 fn store_array_f64x2(self, a: f64x2<Self>, dest: &mut [f64; 2usize]) -> () {
3376 crate::transmute::checked_transmute_store(a.val.0, dest);
3377 }
3378 #[inline(always)]
3379 fn cvt_from_bytes_f64x2(self, a: u8x16<Self>) -> f64x2<Self> {
3380 f64x2 {
3381 val: crate::transmute::checked_transmute_copy(&a.val),
3382 simd: self,
3383 }
3384 }
3385 #[inline(always)]
3386 fn cvt_to_bytes_f64x2(self, a: f64x2<Self>) -> u8x16<Self> {
3387 u8x16 {
3388 val: crate::transmute::checked_transmute_copy(&a.val),
3389 simd: self,
3390 }
3391 }
3392 #[inline(always)]
3393 fn slide_f64x2<const SHIFT: usize>(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3394 if SHIFT >= 2usize {
3395 return b;
3396 }
3397 let result = dyn_alignr_128(
3398 self,
3399 self.cvt_to_bytes_f64x2(b).val.0,
3400 self.cvt_to_bytes_f64x2(a).val.0,
3401 SHIFT * 8usize,
3402 );
3403 self.cvt_from_bytes_f64x2(u8x16 {
3404 val: crate::support::Aligned128(result),
3405 simd: self,
3406 })
3407 }
3408 #[inline(always)]
3409 fn slide_within_blocks_f64x2<const SHIFT: usize>(
3410 self,
3411 a: f64x2<Self>,
3412 b: f64x2<Self>,
3413 ) -> f64x2<Self> {
3414 self.slide_f64x2::<SHIFT>(a, b)
3415 }
3416 #[inline(always)]
3417 fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2<Self>, indices: u8x16<Self>) -> f64x2<Self> {
3418 crate::kernel!(
3419 #[inline(always)]
3420 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, indices: u8x16<Sse4_2>) -> f64x2<Sse4_2> {
3421 let result = _mm_shuffle_epi8(token.cvt_to_bytes_f64x2(a).val.0, indices.into());
3422 token.cvt_from_bytes_f64x2(u8x16 {
3423 val: crate::support::Aligned128(result),
3424 simd: token,
3425 })
3426 }
3427 );
3428 kernel(self, a, indices)
3429 }
3430 #[inline(always)]
3431 fn abs_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
3432 crate::kernel!(
3433 #[inline(always)]
3434 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3435 _mm_andnot_pd(_mm_set1_pd(-0.0), a.into()).simd_into(token)
3436 }
3437 );
3438 kernel(self, a)
3439 }
3440 #[inline(always)]
3441 fn neg_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
3442 crate::kernel!(
3443 #[inline(always)]
3444 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3445 _mm_xor_pd(a.into(), _mm_set1_pd(-0.0)).simd_into(token)
3446 }
3447 );
3448 kernel(self, a)
3449 }
3450 #[inline(always)]
3451 fn sqrt_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
3452 crate::kernel!(
3453 #[inline(always)]
3454 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3455 _mm_sqrt_pd(a.into()).simd_into(token)
3456 }
3457 );
3458 kernel(self, a)
3459 }
3460 #[inline(always)]
3461 fn approximate_recip_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
3462 1.0 / a
3463 }
3464 #[inline(always)]
3465 fn add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3466 crate::kernel!(
3467 #[inline(always)]
3468 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3469 _mm_add_pd(a.into(), b.into()).simd_into(token)
3470 }
3471 );
3472 kernel(self, a, b)
3473 }
3474 #[inline(always)]
3475 fn sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3476 crate::kernel!(
3477 #[inline(always)]
3478 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3479 _mm_sub_pd(a.into(), b.into()).simd_into(token)
3480 }
3481 );
3482 kernel(self, a, b)
3483 }
3484 #[inline(always)]
3485 fn mul_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3486 crate::kernel!(
3487 #[inline(always)]
3488 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3489 _mm_mul_pd(a.into(), b.into()).simd_into(token)
3490 }
3491 );
3492 kernel(self, a, b)
3493 }
3494 #[inline(always)]
3495 fn div_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3496 crate::kernel!(
3497 #[inline(always)]
3498 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3499 _mm_div_pd(a.into(), b.into()).simd_into(token)
3500 }
3501 );
3502 kernel(self, a, b)
3503 }
3504 #[inline(always)]
3505 fn copysign_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3506 crate::kernel!(
3507 #[inline(always)]
3508 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3509 let mask = _mm_set1_pd(-0.0);
3510 _mm_or_pd(_mm_and_pd(mask, b.into()), _mm_andnot_pd(mask, a.into()))
3511 .simd_into(token)
3512 }
3513 );
3514 kernel(self, a, b)
3515 }
3516 #[inline(always)]
3517 fn simd_eq_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
3518 crate::kernel!(
3519 #[inline(always)]
3520 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> mask64x2<Sse4_2> {
3521 _mm_castpd_si128(_mm_cmpeq_pd(a.into(), b.into())).simd_into(token)
3522 }
3523 );
3524 kernel(self, a, b)
3525 }
3526 #[inline(always)]
3527 fn simd_lt_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
3528 crate::kernel!(
3529 #[inline(always)]
3530 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> mask64x2<Sse4_2> {
3531 _mm_castpd_si128(_mm_cmplt_pd(a.into(), b.into())).simd_into(token)
3532 }
3533 );
3534 kernel(self, a, b)
3535 }
3536 #[inline(always)]
3537 fn simd_le_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
3538 crate::kernel!(
3539 #[inline(always)]
3540 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> mask64x2<Sse4_2> {
3541 _mm_castpd_si128(_mm_cmple_pd(a.into(), b.into())).simd_into(token)
3542 }
3543 );
3544 kernel(self, a, b)
3545 }
3546 #[inline(always)]
3547 fn simd_ge_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
3548 crate::kernel!(
3549 #[inline(always)]
3550 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> mask64x2<Sse4_2> {
3551 _mm_castpd_si128(_mm_cmpge_pd(a.into(), b.into())).simd_into(token)
3552 }
3553 );
3554 kernel(self, a, b)
3555 }
3556 #[inline(always)]
3557 fn simd_gt_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
3558 crate::kernel!(
3559 #[inline(always)]
3560 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> mask64x2<Sse4_2> {
3561 _mm_castpd_si128(_mm_cmpgt_pd(a.into(), b.into())).simd_into(token)
3562 }
3563 );
3564 kernel(self, a, b)
3565 }
3566 #[inline(always)]
3567 fn zip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3568 crate::kernel!(
3569 #[inline(always)]
3570 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3571 _mm_unpacklo_pd(a.into(), b.into()).simd_into(token)
3572 }
3573 );
3574 kernel(self, a, b)
3575 }
3576 #[inline(always)]
3577 fn zip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3578 crate::kernel!(
3579 #[inline(always)]
3580 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3581 _mm_unpackhi_pd(a.into(), b.into()).simd_into(token)
3582 }
3583 );
3584 kernel(self, a, b)
3585 }
3586 #[inline(always)]
3587 fn unzip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3588 crate::kernel!(
3589 #[inline(always)]
3590 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3591 _mm_shuffle_pd::<0b00>(a.into(), b.into()).simd_into(token)
3592 }
3593 );
3594 kernel(self, a, b)
3595 }
3596 #[inline(always)]
3597 fn unzip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3598 crate::kernel!(
3599 #[inline(always)]
3600 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3601 _mm_shuffle_pd::<0b11>(a.into(), b.into()).simd_into(token)
3602 }
3603 );
3604 kernel(self, a, b)
3605 }
3606 #[inline(always)]
3607 fn interleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
3608 (self.zip_low_f64x2(a, b), self.zip_high_f64x2(a, b))
3609 }
3610 #[inline(always)]
3611 fn deinterleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
3612 (self.unzip_low_f64x2(a, b), self.unzip_high_f64x2(a, b))
3613 }
3614 #[inline(always)]
3615 fn max_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3616 crate::kernel!(
3617 #[inline(always)]
3618 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3619 _mm_max_pd(a.into(), b.into()).simd_into(token)
3620 }
3621 );
3622 kernel(self, a, b)
3623 }
3624 #[inline(always)]
3625 fn min_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3626 crate::kernel!(
3627 #[inline(always)]
3628 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3629 _mm_min_pd(a.into(), b.into()).simd_into(token)
3630 }
3631 );
3632 kernel(self, a, b)
3633 }
3634 #[inline(always)]
3635 fn max_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3636 crate::kernel!(
3637 #[inline(always)]
3638 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3639 let intermediate = _mm_max_pd(a.into(), b.into());
3640 let b_is_nan = _mm_cmpunord_pd(b.into(), b.into());
3641 _mm_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token)
3642 }
3643 );
3644 kernel(self, a, b)
3645 }
3646 #[inline(always)]
3647 fn min_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
3648 crate::kernel!(
3649 #[inline(always)]
3650 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3651 let intermediate = _mm_min_pd(a.into(), b.into());
3652 let b_is_nan = _mm_cmpunord_pd(b.into(), b.into());
3653 _mm_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token)
3654 }
3655 );
3656 kernel(self, a, b)
3657 }
3658 #[inline(always)]
3659 fn mul_add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
3660 a * b + c
3661 }
3662 #[inline(always)]
3663 fn mul_sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
3664 a * b - c
3665 }
3666 #[inline(always)]
3667 fn floor_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
3668 crate::kernel!(
3669 #[inline(always)]
3670 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3671 _mm_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
3672 .simd_into(token)
3673 }
3674 );
3675 kernel(self, a)
3676 }
3677 #[inline(always)]
3678 fn ceil_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
3679 crate::kernel!(
3680 #[inline(always)]
3681 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3682 _mm_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
3683 .simd_into(token)
3684 }
3685 );
3686 kernel(self, a)
3687 }
3688 #[inline(always)]
3689 fn round_ties_even_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
3690 crate::kernel!(
3691 #[inline(always)]
3692 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3693 _mm_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
3694 .simd_into(token)
3695 }
3696 );
3697 kernel(self, a)
3698 }
3699 #[inline(always)]
3700 fn fract_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
3701 a - self.trunc_f64x2(a)
3702 }
3703 #[inline(always)]
3704 fn trunc_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
3705 crate::kernel!(
3706 #[inline(always)]
3707 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
3708 _mm_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
3709 .simd_into(token)
3710 }
3711 );
3712 kernel(self, a)
3713 }
3714 #[inline(always)]
3715 fn select_f64x2(self, a: mask64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
3716 crate::kernel!(
3717 #[inline(always)]
3718 fn kernel(
3719 token: Sse4_2,
3720 a: mask64x2<Sse4_2>,
3721 b: f64x2<Sse4_2>,
3722 c: f64x2<Sse4_2>,
3723 ) -> f64x2<Sse4_2> {
3724 _mm_blendv_pd(c.into(), b.into(), _mm_castsi128_pd(a.into())).simd_into(token)
3725 }
3726 );
3727 kernel(self, a, b, c)
3728 }
3729 #[inline(always)]
3730 fn combine_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x4<Self> {
3731 f64x4 {
3732 val: crate::support::Aligned256([a.val.0, b.val.0]),
3733 simd: self,
3734 }
3735 }
3736 #[inline(always)]
3737 fn reinterpret_f32_f64x2(self, a: f64x2<Self>) -> f32x4<Self> {
3738 crate::kernel!(
3739 #[inline(always)]
3740 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f32x4<Sse4_2> {
3741 _mm_castpd_ps(a.into()).simd_into(token)
3742 }
3743 );
3744 kernel(self, a)
3745 }
3746 #[inline(always)]
3747 fn splat_mask64x2(self, val: bool) -> mask64x2<Self> {
3748 crate::kernel!(
3749 #[inline(always)]
3750 fn kernel(token: Sse4_2, val: bool) -> mask64x2<Sse4_2> {
3751 let val: i64 = if val { !0 } else { 0 };
3752 _mm_set1_epi64x(val).simd_into(token)
3753 }
3754 );
3755 kernel(self, val)
3756 }
3757 #[inline(always)]
3758 fn load_array_mask64x2(self, val: [i64; 2usize]) -> mask64x2<Self> {
3759 mask64x2 {
3760 val: crate::transmute::checked_transmute_copy(&val),
3761 simd: self,
3762 }
3763 }
3764 #[inline(always)]
3765 fn as_array_mask64x2(self, a: mask64x2<Self>) -> [i64; 2usize] {
3766 crate::transmute::checked_transmute_copy::<__m128i, [i64; 2usize]>(&a.val.0)
3767 }
3768 #[inline(always)]
3769 fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2<Self> {
3770 crate::kernel!(
3771 #[inline(always)]
3772 fn kernel(token: Sse4_2, bits: u64) -> mask64x2<Sse4_2> {
3773 {
3774 let bit_lanes = _mm_set1_epi64x(bits.cast_signed());
3775 let bit_mask = _mm_set_epi64x(2, 1);
3776 _mm_cmpeq_epi64(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
3777 }
3778 .simd_into(token)
3779 }
3780 );
3781 kernel(self, bits)
3782 }
3783 #[inline(always)]
3784 fn to_bitmask_mask64x2(self, a: mask64x2<Self>) -> u64 {
3785 crate::kernel!(
3786 #[inline(always)]
3787 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>) -> u64 {
3788 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 as u64
3789 }
3790 );
3791 kernel(self, a)
3792 }
3793 #[inline(always)]
3794 fn set_mask64x2(self, a: &mut mask64x2<Self>, index: usize, value: bool) -> () {
3795 assert!(
3796 index < 2usize,
3797 "mask lane index {index} is out of bounds for {} lanes",
3798 2usize
3799 );
3800 let mut lanes = self.as_array_mask64x2(*a);
3801 lanes[index] = if value { !0 } else { 0 };
3802 *a = self.load_array_mask64x2(lanes);
3803 }
3804 #[inline(always)]
3805 fn and_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
3806 crate::kernel!(
3807 #[inline(always)]
3808 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>, b: mask64x2<Sse4_2>) -> mask64x2<Sse4_2> {
3809 _mm_and_si128(a.into(), b.into()).simd_into(token)
3810 }
3811 );
3812 kernel(self, a, b)
3813 }
3814 #[inline(always)]
3815 fn or_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
3816 crate::kernel!(
3817 #[inline(always)]
3818 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>, b: mask64x2<Sse4_2>) -> mask64x2<Sse4_2> {
3819 _mm_or_si128(a.into(), b.into()).simd_into(token)
3820 }
3821 );
3822 kernel(self, a, b)
3823 }
3824 #[inline(always)]
3825 fn xor_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
3826 crate::kernel!(
3827 #[inline(always)]
3828 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>, b: mask64x2<Sse4_2>) -> mask64x2<Sse4_2> {
3829 _mm_xor_si128(a.into(), b.into()).simd_into(token)
3830 }
3831 );
3832 kernel(self, a, b)
3833 }
3834 #[inline(always)]
3835 fn not_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
3836 self.xor_mask64x2(a, self.splat_mask64x2(true))
3837 }
3838 #[inline(always)]
3839 fn select_mask64x2(
3840 self,
3841 a: mask64x2<Self>,
3842 b: mask64x2<Self>,
3843 c: mask64x2<Self>,
3844 ) -> mask64x2<Self> {
3845 crate::kernel!(
3846 #[inline(always)]
3847 fn kernel(
3848 token: Sse4_2,
3849 a: mask64x2<Sse4_2>,
3850 b: mask64x2<Sse4_2>,
3851 c: mask64x2<Sse4_2>,
3852 ) -> mask64x2<Sse4_2> {
3853 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
3854 }
3855 );
3856 kernel(self, a, b, c)
3857 }
3858 #[inline(always)]
3859 fn simd_eq_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
3860 crate::kernel!(
3861 #[inline(always)]
3862 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>, b: mask64x2<Sse4_2>) -> mask64x2<Sse4_2> {
3863 _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token)
3864 }
3865 );
3866 kernel(self, a, b)
3867 }
3868 #[inline(always)]
3869 fn any_true_mask64x2(self, a: mask64x2<Self>) -> bool {
3870 crate::kernel!(
3871 #[inline(always)]
3872 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>) -> bool {
3873 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0
3874 }
3875 );
3876 kernel(self, a)
3877 }
3878 #[inline(always)]
3879 fn all_true_mask64x2(self, a: mask64x2<Self>) -> bool {
3880 crate::kernel!(
3881 #[inline(always)]
3882 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>) -> bool {
3883 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0b11
3884 }
3885 );
3886 kernel(self, a)
3887 }
3888 #[inline(always)]
3889 fn any_false_mask64x2(self, a: mask64x2<Self>) -> bool {
3890 crate::kernel!(
3891 #[inline(always)]
3892 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>) -> bool {
3893 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0b11
3894 }
3895 );
3896 kernel(self, a)
3897 }
3898 #[inline(always)]
3899 fn all_false_mask64x2(self, a: mask64x2<Self>) -> bool {
3900 crate::kernel!(
3901 #[inline(always)]
3902 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>) -> bool {
3903 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0
3904 }
3905 );
3906 kernel(self, a)
3907 }
3908 #[inline(always)]
3909 fn combine_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x4<Self> {
3910 mask64x4 {
3911 val: crate::support::Aligned256([a.val.0, b.val.0]),
3912 simd: self,
3913 }
3914 }
3915 #[inline(always)]
3916 fn splat_f32x8(self, val: f32) -> f32x8<Self> {
3917 let half = self.splat_f32x4(val);
3918 self.combine_f32x4(half, half)
3919 }
3920 #[inline(always)]
3921 fn load_array_f32x8(self, val: [f32; 8usize]) -> f32x8<Self> {
3922 f32x8 {
3923 val: crate::transmute::checked_transmute_copy(&val),
3924 simd: self,
3925 }
3926 }
3927 #[inline(always)]
3928 fn load_array_ref_f32x8(self, val: &[f32; 8usize]) -> f32x8<Self> {
3929 f32x8 {
3930 val: crate::transmute::checked_transmute_copy(val),
3931 simd: self,
3932 }
3933 }
3934 #[inline(always)]
3935 fn as_array_f32x8(self, a: f32x8<Self>) -> [f32; 8usize] {
3936 crate::transmute::checked_transmute_copy::<[__m128; 2usize], [f32; 8usize]>(&a.val.0)
3937 }
3938 #[inline(always)]
3939 fn as_array_ref_f32x8(self, a: &f32x8<Self>) -> &[f32; 8usize] {
3940 crate::transmute::checked_cast_ref::<[__m128; 2usize], [f32; 8usize]>(&a.val.0)
3941 }
3942 #[inline(always)]
3943 fn as_array_mut_f32x8(self, a: &mut f32x8<Self>) -> &mut [f32; 8usize] {
3944 crate::transmute::checked_cast_mut::<[__m128; 2usize], [f32; 8usize]>(&mut a.val.0)
3945 }
3946 #[inline(always)]
3947 fn store_array_f32x8(self, a: f32x8<Self>, dest: &mut [f32; 8usize]) -> () {
3948 crate::transmute::checked_transmute_store(a.val.0, dest);
3949 }
3950 #[inline(always)]
3951 fn cvt_from_bytes_f32x8(self, a: u8x32<Self>) -> f32x8<Self> {
3952 f32x8 {
3953 val: crate::transmute::checked_transmute_copy(&a.val),
3954 simd: self,
3955 }
3956 }
3957 #[inline(always)]
3958 fn cvt_to_bytes_f32x8(self, a: f32x8<Self>) -> u8x32<Self> {
3959 u8x32 {
3960 val: crate::transmute::checked_transmute_copy(&a.val),
3961 simd: self,
3962 }
3963 }
3964 #[inline(always)]
3965 fn slide_f32x8<const SHIFT: usize>(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
3966 if SHIFT >= 8usize {
3967 return b;
3968 }
3969 let result = cross_block_alignr_128x2(
3970 self,
3971 self.cvt_to_bytes_f32x8(b).val.0,
3972 self.cvt_to_bytes_f32x8(a).val.0,
3973 SHIFT * 4usize,
3974 );
3975 self.cvt_from_bytes_f32x8(u8x32 {
3976 val: crate::support::Aligned256(result),
3977 simd: self,
3978 })
3979 }
3980 #[inline(always)]
3981 fn slide_within_blocks_f32x8<const SHIFT: usize>(
3982 self,
3983 a: f32x8<Self>,
3984 b: f32x8<Self>,
3985 ) -> f32x8<Self> {
3986 let (a0, a1) = self.split_f32x8(a);
3987 let (b0, b1) = self.split_f32x8(b);
3988 self.combine_f32x4(
3989 self.slide_within_blocks_f32x4::<SHIFT>(a0, b0),
3990 self.slide_within_blocks_f32x4::<SHIFT>(a1, b1),
3991 )
3992 }
3993 #[inline(always)]
3994 fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8<Self>, indices: u8x32<Self>) -> f32x8<Self> {
3995 let (a0, a1) = self.split_f32x8(a);
3996 let (indices0, indices1) = self.split_u8x32(indices);
3997 self.combine_f32x4(
3998 self.swizzle_dyn_within_blocks_f32x4(a0, indices0),
3999 self.swizzle_dyn_within_blocks_f32x4(a1, indices1),
4000 )
4001 }
4002 #[inline(always)]
4003 fn abs_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
4004 let (a0, a1) = self.split_f32x8(a);
4005 self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1))
4006 }
4007 #[inline(always)]
4008 fn neg_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
4009 let (a0, a1) = self.split_f32x8(a);
4010 self.combine_f32x4(self.neg_f32x4(a0), self.neg_f32x4(a1))
4011 }
4012 #[inline(always)]
4013 fn sqrt_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
4014 let (a0, a1) = self.split_f32x8(a);
4015 self.combine_f32x4(self.sqrt_f32x4(a0), self.sqrt_f32x4(a1))
4016 }
4017 #[inline(always)]
4018 fn approximate_recip_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
4019 let (a0, a1) = self.split_f32x8(a);
4020 self.combine_f32x4(
4021 self.approximate_recip_f32x4(a0),
4022 self.approximate_recip_f32x4(a1),
4023 )
4024 }
4025 #[inline(always)]
4026 fn add_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
4027 let (a0, a1) = self.split_f32x8(a);
4028 let (b0, b1) = self.split_f32x8(b);
4029 self.combine_f32x4(self.add_f32x4(a0, b0), self.add_f32x4(a1, b1))
4030 }
4031 #[inline(always)]
4032 fn sub_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
4033 let (a0, a1) = self.split_f32x8(a);
4034 let (b0, b1) = self.split_f32x8(b);
4035 self.combine_f32x4(self.sub_f32x4(a0, b0), self.sub_f32x4(a1, b1))
4036 }
4037 #[inline(always)]
4038 fn mul_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
4039 let (a0, a1) = self.split_f32x8(a);
4040 let (b0, b1) = self.split_f32x8(b);
4041 self.combine_f32x4(self.mul_f32x4(a0, b0), self.mul_f32x4(a1, b1))
4042 }
4043 #[inline(always)]
4044 fn div_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
4045 let (a0, a1) = self.split_f32x8(a);
4046 let (b0, b1) = self.split_f32x8(b);
4047 self.combine_f32x4(self.div_f32x4(a0, b0), self.div_f32x4(a1, b1))
4048 }
4049 #[inline(always)]
4050 fn copysign_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
4051 let (a0, a1) = self.split_f32x8(a);
4052 let (b0, b1) = self.split_f32x8(b);
4053 self.combine_f32x4(self.copysign_f32x4(a0, b0), self.copysign_f32x4(a1, b1))
4054 }
4055 #[inline(always)]
4056 fn simd_eq_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
4057 let (a0, a1) = self.split_f32x8(a);
4058 let (b0, b1) = self.split_f32x8(b);
4059 self.combine_mask32x4(self.simd_eq_f32x4(a0, b0), self.simd_eq_f32x4(a1, b1))
4060 }
4061 #[inline(always)]
4062 fn simd_lt_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
4063 let (a0, a1) = self.split_f32x8(a);
4064 let (b0, b1) = self.split_f32x8(b);
4065 self.combine_mask32x4(self.simd_lt_f32x4(a0, b0), self.simd_lt_f32x4(a1, b1))
4066 }
4067 #[inline(always)]
4068 fn simd_le_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
4069 let (a0, a1) = self.split_f32x8(a);
4070 let (b0, b1) = self.split_f32x8(b);
4071 self.combine_mask32x4(self.simd_le_f32x4(a0, b0), self.simd_le_f32x4(a1, b1))
4072 }
4073 #[inline(always)]
4074 fn simd_ge_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
4075 let (a0, a1) = self.split_f32x8(a);
4076 let (b0, b1) = self.split_f32x8(b);
4077 self.combine_mask32x4(self.simd_ge_f32x4(a0, b0), self.simd_ge_f32x4(a1, b1))
4078 }
4079 #[inline(always)]
4080 fn simd_gt_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
4081 let (a0, a1) = self.split_f32x8(a);
4082 let (b0, b1) = self.split_f32x8(b);
4083 self.combine_mask32x4(self.simd_gt_f32x4(a0, b0), self.simd_gt_f32x4(a1, b1))
4084 }
4085 #[inline(always)]
4086 fn zip_low_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
4087 let (a0, _) = self.split_f32x8(a);
4088 let (b0, _) = self.split_f32x8(b);
4089 self.combine_f32x4(self.zip_low_f32x4(a0, b0), self.zip_high_f32x4(a0, b0))
4090 }
4091 #[inline(always)]
4092 fn zip_high_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
4093 let (_, a1) = self.split_f32x8(a);
4094 let (_, b1) = self.split_f32x8(b);
4095 self.combine_f32x4(self.zip_low_f32x4(a1, b1), self.zip_high_f32x4(a1, b1))
4096 }
4097 #[inline(always)]
4098 fn unzip_low_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
4099 let (a0, a1) = self.split_f32x8(a);
4100 let (b0, b1) = self.split_f32x8(b);
4101 self.combine_f32x4(self.unzip_low_f32x4(a0, a1), self.unzip_low_f32x4(b0, b1))
4102 }
4103 #[inline(always)]
4104 fn unzip_high_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
4105 let (a0, a1) = self.split_f32x8(a);
4106 let (b0, b1) = self.split_f32x8(b);
4107 self.combine_f32x4(self.unzip_high_f32x4(a0, a1), self.unzip_high_f32x4(b0, b1))
4108 }
4109 #[inline(always)]
4110 fn interleave_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> (f32x8<Self>, f32x8<Self>) {
4111 let (a0, a1) = self.split_f32x8(a);
4112 let (b0, b1) = self.split_f32x8(b);
4113 let lo_lo = self.zip_low_f32x4(a0, b0);
4114 let lo_hi = self.zip_high_f32x4(a0, b0);
4115 let hi_lo = self.zip_low_f32x4(a1, b1);
4116 let hi_hi = self.zip_high_f32x4(a1, b1);
4117 (
4118 self.combine_f32x4(lo_lo, lo_hi),
4119 self.combine_f32x4(hi_lo, hi_hi),
4120 )
4121 }
4122 #[inline(always)]
4123 fn deinterleave_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> (f32x8<Self>, f32x8<Self>) {
4124 let (a0, a1) = self.split_f32x8(a);
4125 let (b0, b1) = self.split_f32x8(b);
4126 let lo_even = self.unzip_low_f32x4(a0, a1);
4127 let lo_odd = self.unzip_high_f32x4(a0, a1);
4128 let hi_even = self.unzip_low_f32x4(b0, b1);
4129 let hi_odd = self.unzip_high_f32x4(b0, b1);
4130 (
4131 self.combine_f32x4(lo_even, hi_even),
4132 self.combine_f32x4(lo_odd, hi_odd),
4133 )
4134 }
4135 #[inline(always)]
4136 fn max_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
4137 let (a0, a1) = self.split_f32x8(a);
4138 let (b0, b1) = self.split_f32x8(b);
4139 self.combine_f32x4(self.max_f32x4(a0, b0), self.max_f32x4(a1, b1))
4140 }
4141 #[inline(always)]
4142 fn min_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
4143 let (a0, a1) = self.split_f32x8(a);
4144 let (b0, b1) = self.split_f32x8(b);
4145 self.combine_f32x4(self.min_f32x4(a0, b0), self.min_f32x4(a1, b1))
4146 }
4147 #[inline(always)]
4148 fn max_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
4149 let (a0, a1) = self.split_f32x8(a);
4150 let (b0, b1) = self.split_f32x8(b);
4151 self.combine_f32x4(
4152 self.max_precise_f32x4(a0, b0),
4153 self.max_precise_f32x4(a1, b1),
4154 )
4155 }
4156 #[inline(always)]
4157 fn min_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
4158 let (a0, a1) = self.split_f32x8(a);
4159 let (b0, b1) = self.split_f32x8(b);
4160 self.combine_f32x4(
4161 self.min_precise_f32x4(a0, b0),
4162 self.min_precise_f32x4(a1, b1),
4163 )
4164 }
4165 #[inline(always)]
4166 fn mul_add_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
4167 let (a0, a1) = self.split_f32x8(a);
4168 let (b0, b1) = self.split_f32x8(b);
4169 let (c0, c1) = self.split_f32x8(c);
4170 self.combine_f32x4(
4171 self.mul_add_f32x4(a0, b0, c0),
4172 self.mul_add_f32x4(a1, b1, c1),
4173 )
4174 }
4175 #[inline(always)]
4176 fn mul_sub_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
4177 let (a0, a1) = self.split_f32x8(a);
4178 let (b0, b1) = self.split_f32x8(b);
4179 let (c0, c1) = self.split_f32x8(c);
4180 self.combine_f32x4(
4181 self.mul_sub_f32x4(a0, b0, c0),
4182 self.mul_sub_f32x4(a1, b1, c1),
4183 )
4184 }
4185 #[inline(always)]
4186 fn floor_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
4187 let (a0, a1) = self.split_f32x8(a);
4188 self.combine_f32x4(self.floor_f32x4(a0), self.floor_f32x4(a1))
4189 }
4190 #[inline(always)]
4191 fn ceil_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
4192 let (a0, a1) = self.split_f32x8(a);
4193 self.combine_f32x4(self.ceil_f32x4(a0), self.ceil_f32x4(a1))
4194 }
4195 #[inline(always)]
4196 fn round_ties_even_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
4197 let (a0, a1) = self.split_f32x8(a);
4198 self.combine_f32x4(
4199 self.round_ties_even_f32x4(a0),
4200 self.round_ties_even_f32x4(a1),
4201 )
4202 }
4203 #[inline(always)]
4204 fn fract_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
4205 let (a0, a1) = self.split_f32x8(a);
4206 self.combine_f32x4(self.fract_f32x4(a0), self.fract_f32x4(a1))
4207 }
4208 #[inline(always)]
4209 fn trunc_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
4210 let (a0, a1) = self.split_f32x8(a);
4211 self.combine_f32x4(self.trunc_f32x4(a0), self.trunc_f32x4(a1))
4212 }
4213 #[inline(always)]
4214 fn select_f32x8(self, a: mask32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
4215 let (a0, a1) = self.split_mask32x8(a);
4216 let (b0, b1) = self.split_f32x8(b);
4217 let (c0, c1) = self.split_f32x8(c);
4218 self.combine_f32x4(self.select_f32x4(a0, b0, c0), self.select_f32x4(a1, b1, c1))
4219 }
4220 #[inline(always)]
4221 fn combine_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x16<Self> {
4222 f32x16 {
4223 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
4224 simd: self,
4225 }
4226 }
4227 #[inline(always)]
4228 fn split_f32x8(self, a: f32x8<Self>) -> (f32x4<Self>, f32x4<Self>) {
4229 (
4230 f32x4 {
4231 val: crate::support::Aligned128(a.val.0[0]),
4232 simd: self,
4233 },
4234 f32x4 {
4235 val: crate::support::Aligned128(a.val.0[1]),
4236 simd: self,
4237 },
4238 )
4239 }
4240 #[inline(always)]
4241 fn reinterpret_f64_f32x8(self, a: f32x8<Self>) -> f64x4<Self> {
4242 let (a0, a1) = self.split_f32x8(a);
4243 self.combine_f64x2(
4244 self.reinterpret_f64_f32x4(a0),
4245 self.reinterpret_f64_f32x4(a1),
4246 )
4247 }
4248 #[inline(always)]
4249 fn reinterpret_i32_f32x8(self, a: f32x8<Self>) -> i32x8<Self> {
4250 let (a0, a1) = self.split_f32x8(a);
4251 self.combine_i32x4(
4252 self.reinterpret_i32_f32x4(a0),
4253 self.reinterpret_i32_f32x4(a1),
4254 )
4255 }
4256 #[inline(always)]
4257 fn reinterpret_u8_f32x8(self, a: f32x8<Self>) -> u8x32<Self> {
4258 let (a0, a1) = self.split_f32x8(a);
4259 self.combine_u8x16(self.reinterpret_u8_f32x4(a0), self.reinterpret_u8_f32x4(a1))
4260 }
4261 #[inline(always)]
4262 fn reinterpret_u32_f32x8(self, a: f32x8<Self>) -> u32x8<Self> {
4263 let (a0, a1) = self.split_f32x8(a);
4264 self.combine_u32x4(
4265 self.reinterpret_u32_f32x4(a0),
4266 self.reinterpret_u32_f32x4(a1),
4267 )
4268 }
4269 #[inline(always)]
4270 fn cvt_u32_f32x8(self, a: f32x8<Self>) -> u32x8<Self> {
4271 let (a0, a1) = self.split_f32x8(a);
4272 self.combine_u32x4(self.cvt_u32_f32x4(a0), self.cvt_u32_f32x4(a1))
4273 }
4274 #[inline(always)]
4275 fn cvt_u32_precise_f32x8(self, a: f32x8<Self>) -> u32x8<Self> {
4276 let (a0, a1) = self.split_f32x8(a);
4277 self.combine_u32x4(
4278 self.cvt_u32_precise_f32x4(a0),
4279 self.cvt_u32_precise_f32x4(a1),
4280 )
4281 }
4282 #[inline(always)]
4283 fn cvt_i32_f32x8(self, a: f32x8<Self>) -> i32x8<Self> {
4284 let (a0, a1) = self.split_f32x8(a);
4285 self.combine_i32x4(self.cvt_i32_f32x4(a0), self.cvt_i32_f32x4(a1))
4286 }
4287 #[inline(always)]
4288 fn cvt_i32_precise_f32x8(self, a: f32x8<Self>) -> i32x8<Self> {
4289 let (a0, a1) = self.split_f32x8(a);
4290 self.combine_i32x4(
4291 self.cvt_i32_precise_f32x4(a0),
4292 self.cvt_i32_precise_f32x4(a1),
4293 )
4294 }
4295 #[inline(always)]
4296 fn splat_i8x32(self, val: i8) -> i8x32<Self> {
4297 let half = self.splat_i8x16(val);
4298 self.combine_i8x16(half, half)
4299 }
4300 #[inline(always)]
4301 fn load_array_i8x32(self, val: [i8; 32usize]) -> i8x32<Self> {
4302 i8x32 {
4303 val: crate::transmute::checked_transmute_copy(&val),
4304 simd: self,
4305 }
4306 }
4307 #[inline(always)]
4308 fn load_array_ref_i8x32(self, val: &[i8; 32usize]) -> i8x32<Self> {
4309 i8x32 {
4310 val: crate::transmute::checked_transmute_copy(val),
4311 simd: self,
4312 }
4313 }
4314 #[inline(always)]
4315 fn as_array_i8x32(self, a: i8x32<Self>) -> [i8; 32usize] {
4316 crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i8; 32usize]>(&a.val.0)
4317 }
4318 #[inline(always)]
4319 fn as_array_ref_i8x32(self, a: &i8x32<Self>) -> &[i8; 32usize] {
4320 crate::transmute::checked_cast_ref::<[__m128i; 2usize], [i8; 32usize]>(&a.val.0)
4321 }
4322 #[inline(always)]
4323 fn as_array_mut_i8x32(self, a: &mut i8x32<Self>) -> &mut [i8; 32usize] {
4324 crate::transmute::checked_cast_mut::<[__m128i; 2usize], [i8; 32usize]>(&mut a.val.0)
4325 }
4326 #[inline(always)]
4327 fn store_array_i8x32(self, a: i8x32<Self>, dest: &mut [i8; 32usize]) -> () {
4328 crate::transmute::checked_transmute_store(a.val.0, dest);
4329 }
4330 #[inline(always)]
4331 fn cvt_from_bytes_i8x32(self, a: u8x32<Self>) -> i8x32<Self> {
4332 i8x32 {
4333 val: crate::transmute::checked_transmute_copy(&a.val),
4334 simd: self,
4335 }
4336 }
4337 #[inline(always)]
4338 fn cvt_to_bytes_i8x32(self, a: i8x32<Self>) -> u8x32<Self> {
4339 u8x32 {
4340 val: crate::transmute::checked_transmute_copy(&a.val),
4341 simd: self,
4342 }
4343 }
4344 #[inline(always)]
4345 fn slide_i8x32<const SHIFT: usize>(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4346 if SHIFT >= 32usize {
4347 return b;
4348 }
4349 let result = cross_block_alignr_128x2(
4350 self,
4351 self.cvt_to_bytes_i8x32(b).val.0,
4352 self.cvt_to_bytes_i8x32(a).val.0,
4353 SHIFT,
4354 );
4355 self.cvt_from_bytes_i8x32(u8x32 {
4356 val: crate::support::Aligned256(result),
4357 simd: self,
4358 })
4359 }
4360 #[inline(always)]
4361 fn slide_within_blocks_i8x32<const SHIFT: usize>(
4362 self,
4363 a: i8x32<Self>,
4364 b: i8x32<Self>,
4365 ) -> i8x32<Self> {
4366 let (a0, a1) = self.split_i8x32(a);
4367 let (b0, b1) = self.split_i8x32(b);
4368 self.combine_i8x16(
4369 self.slide_within_blocks_i8x16::<SHIFT>(a0, b0),
4370 self.slide_within_blocks_i8x16::<SHIFT>(a1, b1),
4371 )
4372 }
4373 #[inline(always)]
4374 fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32<Self>, indices: u8x32<Self>) -> i8x32<Self> {
4375 let (a0, a1) = self.split_i8x32(a);
4376 let (indices0, indices1) = self.split_u8x32(indices);
4377 self.combine_i8x16(
4378 self.swizzle_dyn_within_blocks_i8x16(a0, indices0),
4379 self.swizzle_dyn_within_blocks_i8x16(a1, indices1),
4380 )
4381 }
4382 #[inline(always)]
4383 fn add_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4384 let (a0, a1) = self.split_i8x32(a);
4385 let (b0, b1) = self.split_i8x32(b);
4386 self.combine_i8x16(self.add_i8x16(a0, b0), self.add_i8x16(a1, b1))
4387 }
4388 #[inline(always)]
4389 fn sub_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4390 let (a0, a1) = self.split_i8x32(a);
4391 let (b0, b1) = self.split_i8x32(b);
4392 self.combine_i8x16(self.sub_i8x16(a0, b0), self.sub_i8x16(a1, b1))
4393 }
4394 #[inline(always)]
4395 fn mul_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4396 let (a0, a1) = self.split_i8x32(a);
4397 let (b0, b1) = self.split_i8x32(b);
4398 self.combine_i8x16(self.mul_i8x16(a0, b0), self.mul_i8x16(a1, b1))
4399 }
4400 #[inline(always)]
4401 fn and_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4402 let (a0, a1) = self.split_i8x32(a);
4403 let (b0, b1) = self.split_i8x32(b);
4404 self.combine_i8x16(self.and_i8x16(a0, b0), self.and_i8x16(a1, b1))
4405 }
4406 #[inline(always)]
4407 fn or_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4408 let (a0, a1) = self.split_i8x32(a);
4409 let (b0, b1) = self.split_i8x32(b);
4410 self.combine_i8x16(self.or_i8x16(a0, b0), self.or_i8x16(a1, b1))
4411 }
4412 #[inline(always)]
4413 fn xor_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4414 let (a0, a1) = self.split_i8x32(a);
4415 let (b0, b1) = self.split_i8x32(b);
4416 self.combine_i8x16(self.xor_i8x16(a0, b0), self.xor_i8x16(a1, b1))
4417 }
4418 #[inline(always)]
4419 fn not_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
4420 let (a0, a1) = self.split_i8x32(a);
4421 self.combine_i8x16(self.not_i8x16(a0), self.not_i8x16(a1))
4422 }
4423 #[inline(always)]
4424 fn shl_i8x32(self, a: i8x32<Self>, shift: u32) -> i8x32<Self> {
4425 let (a0, a1) = self.split_i8x32(a);
4426 self.combine_i8x16(self.shl_i8x16(a0, shift), self.shl_i8x16(a1, shift))
4427 }
4428 #[inline(always)]
4429 fn shlv_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4430 let (a0, a1) = self.split_i8x32(a);
4431 let (b0, b1) = self.split_i8x32(b);
4432 self.combine_i8x16(self.shlv_i8x16(a0, b0), self.shlv_i8x16(a1, b1))
4433 }
4434 #[inline(always)]
4435 fn shr_i8x32(self, a: i8x32<Self>, shift: u32) -> i8x32<Self> {
4436 let (a0, a1) = self.split_i8x32(a);
4437 self.combine_i8x16(self.shr_i8x16(a0, shift), self.shr_i8x16(a1, shift))
4438 }
4439 #[inline(always)]
4440 fn shrv_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4441 let (a0, a1) = self.split_i8x32(a);
4442 let (b0, b1) = self.split_i8x32(b);
4443 self.combine_i8x16(self.shrv_i8x16(a0, b0), self.shrv_i8x16(a1, b1))
4444 }
4445 #[inline(always)]
4446 fn simd_eq_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
4447 let (a0, a1) = self.split_i8x32(a);
4448 let (b0, b1) = self.split_i8x32(b);
4449 self.combine_mask8x16(self.simd_eq_i8x16(a0, b0), self.simd_eq_i8x16(a1, b1))
4450 }
4451 #[inline(always)]
4452 fn simd_lt_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
4453 let (a0, a1) = self.split_i8x32(a);
4454 let (b0, b1) = self.split_i8x32(b);
4455 self.combine_mask8x16(self.simd_lt_i8x16(a0, b0), self.simd_lt_i8x16(a1, b1))
4456 }
4457 #[inline(always)]
4458 fn simd_le_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
4459 let (a0, a1) = self.split_i8x32(a);
4460 let (b0, b1) = self.split_i8x32(b);
4461 self.combine_mask8x16(self.simd_le_i8x16(a0, b0), self.simd_le_i8x16(a1, b1))
4462 }
4463 #[inline(always)]
4464 fn simd_ge_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
4465 let (a0, a1) = self.split_i8x32(a);
4466 let (b0, b1) = self.split_i8x32(b);
4467 self.combine_mask8x16(self.simd_ge_i8x16(a0, b0), self.simd_ge_i8x16(a1, b1))
4468 }
4469 #[inline(always)]
4470 fn simd_gt_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
4471 let (a0, a1) = self.split_i8x32(a);
4472 let (b0, b1) = self.split_i8x32(b);
4473 self.combine_mask8x16(self.simd_gt_i8x16(a0, b0), self.simd_gt_i8x16(a1, b1))
4474 }
4475 #[inline(always)]
4476 fn zip_low_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4477 let (a0, _) = self.split_i8x32(a);
4478 let (b0, _) = self.split_i8x32(b);
4479 self.combine_i8x16(self.zip_low_i8x16(a0, b0), self.zip_high_i8x16(a0, b0))
4480 }
4481 #[inline(always)]
4482 fn zip_high_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4483 let (_, a1) = self.split_i8x32(a);
4484 let (_, b1) = self.split_i8x32(b);
4485 self.combine_i8x16(self.zip_low_i8x16(a1, b1), self.zip_high_i8x16(a1, b1))
4486 }
4487 #[inline(always)]
4488 fn unzip_low_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4489 let (a0, a1) = self.split_i8x32(a);
4490 let (b0, b1) = self.split_i8x32(b);
4491 self.combine_i8x16(self.unzip_low_i8x16(a0, a1), self.unzip_low_i8x16(b0, b1))
4492 }
4493 #[inline(always)]
4494 fn unzip_high_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4495 let (a0, a1) = self.split_i8x32(a);
4496 let (b0, b1) = self.split_i8x32(b);
4497 self.combine_i8x16(self.unzip_high_i8x16(a0, a1), self.unzip_high_i8x16(b0, b1))
4498 }
4499 #[inline(always)]
4500 fn interleave_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i8x32<Self>, i8x32<Self>) {
4501 let (a0, a1) = self.split_i8x32(a);
4502 let (b0, b1) = self.split_i8x32(b);
4503 let lo_lo = self.zip_low_i8x16(a0, b0);
4504 let lo_hi = self.zip_high_i8x16(a0, b0);
4505 let hi_lo = self.zip_low_i8x16(a1, b1);
4506 let hi_hi = self.zip_high_i8x16(a1, b1);
4507 (
4508 self.combine_i8x16(lo_lo, lo_hi),
4509 self.combine_i8x16(hi_lo, hi_hi),
4510 )
4511 }
4512 #[inline(always)]
4513 fn deinterleave_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i8x32<Self>, i8x32<Self>) {
4514 let (a0, a1) = self.split_i8x32(a);
4515 let (b0, b1) = self.split_i8x32(b);
4516 let lo_even = self.unzip_low_i8x16(a0, a1);
4517 let lo_odd = self.unzip_high_i8x16(a0, a1);
4518 let hi_even = self.unzip_low_i8x16(b0, b1);
4519 let hi_odd = self.unzip_high_i8x16(b0, b1);
4520 (
4521 self.combine_i8x16(lo_even, hi_even),
4522 self.combine_i8x16(lo_odd, hi_odd),
4523 )
4524 }
4525 #[inline(always)]
4526 fn select_i8x32(self, a: mask8x32<Self>, b: i8x32<Self>, c: i8x32<Self>) -> i8x32<Self> {
4527 let (a0, a1) = self.split_mask8x32(a);
4528 let (b0, b1) = self.split_i8x32(b);
4529 let (c0, c1) = self.split_i8x32(c);
4530 self.combine_i8x16(self.select_i8x16(a0, b0, c0), self.select_i8x16(a1, b1, c1))
4531 }
4532 #[inline(always)]
4533 fn min_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4534 let (a0, a1) = self.split_i8x32(a);
4535 let (b0, b1) = self.split_i8x32(b);
4536 self.combine_i8x16(self.min_i8x16(a0, b0), self.min_i8x16(a1, b1))
4537 }
4538 #[inline(always)]
4539 fn max_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
4540 let (a0, a1) = self.split_i8x32(a);
4541 let (b0, b1) = self.split_i8x32(b);
4542 self.combine_i8x16(self.max_i8x16(a0, b0), self.max_i8x16(a1, b1))
4543 }
4544 #[inline(always)]
4545 fn combine_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x64<Self> {
4546 i8x64 {
4547 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
4548 simd: self,
4549 }
4550 }
4551 #[inline(always)]
4552 fn split_i8x32(self, a: i8x32<Self>) -> (i8x16<Self>, i8x16<Self>) {
4553 (
4554 i8x16 {
4555 val: crate::support::Aligned128(a.val.0[0]),
4556 simd: self,
4557 },
4558 i8x16 {
4559 val: crate::support::Aligned128(a.val.0[1]),
4560 simd: self,
4561 },
4562 )
4563 }
4564 #[inline(always)]
4565 fn neg_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
4566 let (a0, a1) = self.split_i8x32(a);
4567 self.combine_i8x16(self.neg_i8x16(a0), self.neg_i8x16(a1))
4568 }
4569 #[inline(always)]
4570 fn reinterpret_u8_i8x32(self, a: i8x32<Self>) -> u8x32<Self> {
4571 let (a0, a1) = self.split_i8x32(a);
4572 self.combine_u8x16(self.reinterpret_u8_i8x16(a0), self.reinterpret_u8_i8x16(a1))
4573 }
4574 #[inline(always)]
4575 fn reinterpret_u32_i8x32(self, a: i8x32<Self>) -> u32x8<Self> {
4576 let (a0, a1) = self.split_i8x32(a);
4577 self.combine_u32x4(
4578 self.reinterpret_u32_i8x16(a0),
4579 self.reinterpret_u32_i8x16(a1),
4580 )
4581 }
4582 #[inline(always)]
4583 fn splat_u8x32(self, val: u8) -> u8x32<Self> {
4584 let half = self.splat_u8x16(val);
4585 self.combine_u8x16(half, half)
4586 }
4587 #[inline(always)]
4588 fn load_array_u8x32(self, val: [u8; 32usize]) -> u8x32<Self> {
4589 u8x32 {
4590 val: crate::transmute::checked_transmute_copy(&val),
4591 simd: self,
4592 }
4593 }
4594 #[inline(always)]
4595 fn load_array_ref_u8x32(self, val: &[u8; 32usize]) -> u8x32<Self> {
4596 u8x32 {
4597 val: crate::transmute::checked_transmute_copy(val),
4598 simd: self,
4599 }
4600 }
4601 #[inline(always)]
4602 fn as_array_u8x32(self, a: u8x32<Self>) -> [u8; 32usize] {
4603 crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [u8; 32usize]>(&a.val.0)
4604 }
4605 #[inline(always)]
4606 fn as_array_ref_u8x32(self, a: &u8x32<Self>) -> &[u8; 32usize] {
4607 crate::transmute::checked_cast_ref::<[__m128i; 2usize], [u8; 32usize]>(&a.val.0)
4608 }
4609 #[inline(always)]
4610 fn as_array_mut_u8x32(self, a: &mut u8x32<Self>) -> &mut [u8; 32usize] {
4611 crate::transmute::checked_cast_mut::<[__m128i; 2usize], [u8; 32usize]>(&mut a.val.0)
4612 }
4613 #[inline(always)]
4614 fn store_array_u8x32(self, a: u8x32<Self>, dest: &mut [u8; 32usize]) -> () {
4615 crate::transmute::checked_transmute_store(a.val.0, dest);
4616 }
4617 #[inline(always)]
4618 fn cvt_from_bytes_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
4619 u8x32 {
4620 val: crate::transmute::checked_transmute_copy(&a.val),
4621 simd: self,
4622 }
4623 }
4624 #[inline(always)]
4625 fn cvt_to_bytes_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
4626 u8x32 {
4627 val: crate::transmute::checked_transmute_copy(&a.val),
4628 simd: self,
4629 }
4630 }
4631 #[inline(always)]
4632 fn slide_u8x32<const SHIFT: usize>(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4633 if SHIFT >= 32usize {
4634 return b;
4635 }
4636 let result = cross_block_alignr_128x2(
4637 self,
4638 self.cvt_to_bytes_u8x32(b).val.0,
4639 self.cvt_to_bytes_u8x32(a).val.0,
4640 SHIFT,
4641 );
4642 self.cvt_from_bytes_u8x32(u8x32 {
4643 val: crate::support::Aligned256(result),
4644 simd: self,
4645 })
4646 }
4647 #[inline(always)]
4648 fn slide_within_blocks_u8x32<const SHIFT: usize>(
4649 self,
4650 a: u8x32<Self>,
4651 b: u8x32<Self>,
4652 ) -> u8x32<Self> {
4653 let (a0, a1) = self.split_u8x32(a);
4654 let (b0, b1) = self.split_u8x32(b);
4655 self.combine_u8x16(
4656 self.slide_within_blocks_u8x16::<SHIFT>(a0, b0),
4657 self.slide_within_blocks_u8x16::<SHIFT>(a1, b1),
4658 )
4659 }
4660 #[inline(always)]
4661 fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
4662 let (a0, a1) = self.split_u8x32(a);
4663 let (indices0, indices1) = self.split_u8x32(indices);
4664 self.combine_u8x16(
4665 self.swizzle_dyn_within_blocks_u8x16(a0, indices0),
4666 self.swizzle_dyn_within_blocks_u8x16(a1, indices1),
4667 )
4668 }
4669 #[inline(always)]
4670 fn add_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4671 let (a0, a1) = self.split_u8x32(a);
4672 let (b0, b1) = self.split_u8x32(b);
4673 self.combine_u8x16(self.add_u8x16(a0, b0), self.add_u8x16(a1, b1))
4674 }
4675 #[inline(always)]
4676 fn sub_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4677 let (a0, a1) = self.split_u8x32(a);
4678 let (b0, b1) = self.split_u8x32(b);
4679 self.combine_u8x16(self.sub_u8x16(a0, b0), self.sub_u8x16(a1, b1))
4680 }
4681 #[inline(always)]
4682 fn mul_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4683 let (a0, a1) = self.split_u8x32(a);
4684 let (b0, b1) = self.split_u8x32(b);
4685 self.combine_u8x16(self.mul_u8x16(a0, b0), self.mul_u8x16(a1, b1))
4686 }
4687 #[inline(always)]
4688 fn and_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4689 let (a0, a1) = self.split_u8x32(a);
4690 let (b0, b1) = self.split_u8x32(b);
4691 self.combine_u8x16(self.and_u8x16(a0, b0), self.and_u8x16(a1, b1))
4692 }
4693 #[inline(always)]
4694 fn or_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4695 let (a0, a1) = self.split_u8x32(a);
4696 let (b0, b1) = self.split_u8x32(b);
4697 self.combine_u8x16(self.or_u8x16(a0, b0), self.or_u8x16(a1, b1))
4698 }
4699 #[inline(always)]
4700 fn xor_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4701 let (a0, a1) = self.split_u8x32(a);
4702 let (b0, b1) = self.split_u8x32(b);
4703 self.combine_u8x16(self.xor_u8x16(a0, b0), self.xor_u8x16(a1, b1))
4704 }
4705 #[inline(always)]
4706 fn not_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
4707 let (a0, a1) = self.split_u8x32(a);
4708 self.combine_u8x16(self.not_u8x16(a0), self.not_u8x16(a1))
4709 }
4710 #[inline(always)]
4711 fn shl_u8x32(self, a: u8x32<Self>, shift: u32) -> u8x32<Self> {
4712 let (a0, a1) = self.split_u8x32(a);
4713 self.combine_u8x16(self.shl_u8x16(a0, shift), self.shl_u8x16(a1, shift))
4714 }
4715 #[inline(always)]
4716 fn shlv_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4717 let (a0, a1) = self.split_u8x32(a);
4718 let (b0, b1) = self.split_u8x32(b);
4719 self.combine_u8x16(self.shlv_u8x16(a0, b0), self.shlv_u8x16(a1, b1))
4720 }
4721 #[inline(always)]
4722 fn shr_u8x32(self, a: u8x32<Self>, shift: u32) -> u8x32<Self> {
4723 let (a0, a1) = self.split_u8x32(a);
4724 self.combine_u8x16(self.shr_u8x16(a0, shift), self.shr_u8x16(a1, shift))
4725 }
4726 #[inline(always)]
4727 fn shrv_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4728 let (a0, a1) = self.split_u8x32(a);
4729 let (b0, b1) = self.split_u8x32(b);
4730 self.combine_u8x16(self.shrv_u8x16(a0, b0), self.shrv_u8x16(a1, b1))
4731 }
4732 #[inline(always)]
4733 fn simd_eq_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
4734 let (a0, a1) = self.split_u8x32(a);
4735 let (b0, b1) = self.split_u8x32(b);
4736 self.combine_mask8x16(self.simd_eq_u8x16(a0, b0), self.simd_eq_u8x16(a1, b1))
4737 }
4738 #[inline(always)]
4739 fn simd_lt_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
4740 let (a0, a1) = self.split_u8x32(a);
4741 let (b0, b1) = self.split_u8x32(b);
4742 self.combine_mask8x16(self.simd_lt_u8x16(a0, b0), self.simd_lt_u8x16(a1, b1))
4743 }
4744 #[inline(always)]
4745 fn simd_le_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
4746 let (a0, a1) = self.split_u8x32(a);
4747 let (b0, b1) = self.split_u8x32(b);
4748 self.combine_mask8x16(self.simd_le_u8x16(a0, b0), self.simd_le_u8x16(a1, b1))
4749 }
4750 #[inline(always)]
4751 fn simd_ge_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
4752 let (a0, a1) = self.split_u8x32(a);
4753 let (b0, b1) = self.split_u8x32(b);
4754 self.combine_mask8x16(self.simd_ge_u8x16(a0, b0), self.simd_ge_u8x16(a1, b1))
4755 }
4756 #[inline(always)]
4757 fn simd_gt_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
4758 let (a0, a1) = self.split_u8x32(a);
4759 let (b0, b1) = self.split_u8x32(b);
4760 self.combine_mask8x16(self.simd_gt_u8x16(a0, b0), self.simd_gt_u8x16(a1, b1))
4761 }
4762 #[inline(always)]
4763 fn zip_low_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4764 let (a0, _) = self.split_u8x32(a);
4765 let (b0, _) = self.split_u8x32(b);
4766 self.combine_u8x16(self.zip_low_u8x16(a0, b0), self.zip_high_u8x16(a0, b0))
4767 }
4768 #[inline(always)]
4769 fn zip_high_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4770 let (_, a1) = self.split_u8x32(a);
4771 let (_, b1) = self.split_u8x32(b);
4772 self.combine_u8x16(self.zip_low_u8x16(a1, b1), self.zip_high_u8x16(a1, b1))
4773 }
4774 #[inline(always)]
4775 fn unzip_low_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4776 let (a0, a1) = self.split_u8x32(a);
4777 let (b0, b1) = self.split_u8x32(b);
4778 self.combine_u8x16(self.unzip_low_u8x16(a0, a1), self.unzip_low_u8x16(b0, b1))
4779 }
4780 #[inline(always)]
4781 fn unzip_high_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4782 let (a0, a1) = self.split_u8x32(a);
4783 let (b0, b1) = self.split_u8x32(b);
4784 self.combine_u8x16(self.unzip_high_u8x16(a0, a1), self.unzip_high_u8x16(b0, b1))
4785 }
4786 #[inline(always)]
4787 fn interleave_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u8x32<Self>, u8x32<Self>) {
4788 let (a0, a1) = self.split_u8x32(a);
4789 let (b0, b1) = self.split_u8x32(b);
4790 let lo_lo = self.zip_low_u8x16(a0, b0);
4791 let lo_hi = self.zip_high_u8x16(a0, b0);
4792 let hi_lo = self.zip_low_u8x16(a1, b1);
4793 let hi_hi = self.zip_high_u8x16(a1, b1);
4794 (
4795 self.combine_u8x16(lo_lo, lo_hi),
4796 self.combine_u8x16(hi_lo, hi_hi),
4797 )
4798 }
4799 #[inline(always)]
4800 fn deinterleave_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u8x32<Self>, u8x32<Self>) {
4801 let (a0, a1) = self.split_u8x32(a);
4802 let (b0, b1) = self.split_u8x32(b);
4803 let lo_even = self.unzip_low_u8x16(a0, a1);
4804 let lo_odd = self.unzip_high_u8x16(a0, a1);
4805 let hi_even = self.unzip_low_u8x16(b0, b1);
4806 let hi_odd = self.unzip_high_u8x16(b0, b1);
4807 (
4808 self.combine_u8x16(lo_even, hi_even),
4809 self.combine_u8x16(lo_odd, hi_odd),
4810 )
4811 }
4812 #[inline(always)]
4813 fn select_u8x32(self, a: mask8x32<Self>, b: u8x32<Self>, c: u8x32<Self>) -> u8x32<Self> {
4814 let (a0, a1) = self.split_mask8x32(a);
4815 let (b0, b1) = self.split_u8x32(b);
4816 let (c0, c1) = self.split_u8x32(c);
4817 self.combine_u8x16(self.select_u8x16(a0, b0, c0), self.select_u8x16(a1, b1, c1))
4818 }
4819 #[inline(always)]
4820 fn min_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4821 let (a0, a1) = self.split_u8x32(a);
4822 let (b0, b1) = self.split_u8x32(b);
4823 self.combine_u8x16(self.min_u8x16(a0, b0), self.min_u8x16(a1, b1))
4824 }
4825 #[inline(always)]
4826 fn max_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
4827 let (a0, a1) = self.split_u8x32(a);
4828 let (b0, b1) = self.split_u8x32(b);
4829 self.combine_u8x16(self.max_u8x16(a0, b0), self.max_u8x16(a1, b1))
4830 }
4831 #[inline(always)]
4832 fn combine_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x64<Self> {
4833 u8x64 {
4834 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
4835 simd: self,
4836 }
4837 }
4838 #[inline(always)]
4839 fn split_u8x32(self, a: u8x32<Self>) -> (u8x16<Self>, u8x16<Self>) {
4840 (
4841 u8x16 {
4842 val: crate::support::Aligned128(a.val.0[0]),
4843 simd: self,
4844 },
4845 u8x16 {
4846 val: crate::support::Aligned128(a.val.0[1]),
4847 simd: self,
4848 },
4849 )
4850 }
4851 #[inline(always)]
4852 fn widen_u8x32(self, a: u8x32<Self>) -> u16x32<Self> {
4853 let (a0, a1) = self.split_u8x32(a);
4854 self.combine_u16x16(self.widen_u8x16(a0), self.widen_u8x16(a1))
4855 }
4856 #[inline(always)]
4857 fn reinterpret_u32_u8x32(self, a: u8x32<Self>) -> u32x8<Self> {
4858 let (a0, a1) = self.split_u8x32(a);
4859 self.combine_u32x4(
4860 self.reinterpret_u32_u8x16(a0),
4861 self.reinterpret_u32_u8x16(a1),
4862 )
4863 }
4864 #[inline(always)]
4865 fn splat_mask8x32(self, val: bool) -> mask8x32<Self> {
4866 let half = self.splat_mask8x16(val);
4867 self.combine_mask8x16(half, half)
4868 }
4869 #[inline(always)]
4870 fn load_array_mask8x32(self, val: [i8; 32usize]) -> mask8x32<Self> {
4871 mask8x32 {
4872 val: crate::transmute::checked_transmute_copy(&val),
4873 simd: self,
4874 }
4875 }
4876 #[inline(always)]
4877 fn as_array_mask8x32(self, a: mask8x32<Self>) -> [i8; 32usize] {
4878 crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i8; 32usize]>(&a.val.0)
4879 }
4880 #[inline(always)]
4881 fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32<Self> {
4882 let lo = self.from_bitmask_mask8x16(bits);
4883 let hi = self.from_bitmask_mask8x16(bits >> 16usize);
4884 self.combine_mask8x16(lo, hi)
4885 }
4886 #[inline(always)]
4887 fn to_bitmask_mask8x32(self, a: mask8x32<Self>) -> u64 {
4888 let (lo, hi) = self.split_mask8x32(a);
4889 let lo = self.to_bitmask_mask8x16(lo);
4890 let hi = self.to_bitmask_mask8x16(hi);
4891 lo | (hi << 16usize)
4892 }
4893 #[inline(always)]
4894 fn set_mask8x32(self, a: &mut mask8x32<Self>, index: usize, value: bool) -> () {
4895 assert!(
4896 index < 32usize,
4897 "mask lane index {index} is out of bounds for {} lanes",
4898 32usize
4899 );
4900 let mut lanes = self.as_array_mask8x32(*a);
4901 lanes[index] = if value { !0 } else { 0 };
4902 *a = self.load_array_mask8x32(lanes);
4903 }
4904 #[inline(always)]
4905 fn and_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
4906 let (a0, a1) = self.split_mask8x32(a);
4907 let (b0, b1) = self.split_mask8x32(b);
4908 self.combine_mask8x16(self.and_mask8x16(a0, b0), self.and_mask8x16(a1, b1))
4909 }
4910 #[inline(always)]
4911 fn or_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
4912 let (a0, a1) = self.split_mask8x32(a);
4913 let (b0, b1) = self.split_mask8x32(b);
4914 self.combine_mask8x16(self.or_mask8x16(a0, b0), self.or_mask8x16(a1, b1))
4915 }
4916 #[inline(always)]
4917 fn xor_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
4918 let (a0, a1) = self.split_mask8x32(a);
4919 let (b0, b1) = self.split_mask8x32(b);
4920 self.combine_mask8x16(self.xor_mask8x16(a0, b0), self.xor_mask8x16(a1, b1))
4921 }
4922 #[inline(always)]
4923 fn not_mask8x32(self, a: mask8x32<Self>) -> mask8x32<Self> {
4924 let (a0, a1) = self.split_mask8x32(a);
4925 self.combine_mask8x16(self.not_mask8x16(a0), self.not_mask8x16(a1))
4926 }
4927 #[inline(always)]
4928 fn select_mask8x32(
4929 self,
4930 a: mask8x32<Self>,
4931 b: mask8x32<Self>,
4932 c: mask8x32<Self>,
4933 ) -> mask8x32<Self> {
4934 let (a0, a1) = self.split_mask8x32(a);
4935 let (b0, b1) = self.split_mask8x32(b);
4936 let (c0, c1) = self.split_mask8x32(c);
4937 self.combine_mask8x16(
4938 self.select_mask8x16(a0, b0, c0),
4939 self.select_mask8x16(a1, b1, c1),
4940 )
4941 }
4942 #[inline(always)]
4943 fn simd_eq_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
4944 let (a0, a1) = self.split_mask8x32(a);
4945 let (b0, b1) = self.split_mask8x32(b);
4946 self.combine_mask8x16(self.simd_eq_mask8x16(a0, b0), self.simd_eq_mask8x16(a1, b1))
4947 }
4948 #[inline(always)]
4949 fn any_true_mask8x32(self, a: mask8x32<Self>) -> bool {
4950 let (a0, a1) = self.split_mask8x32(a);
4951 self.any_true_mask8x16(a0) || self.any_true_mask8x16(a1)
4952 }
4953 #[inline(always)]
4954 fn all_true_mask8x32(self, a: mask8x32<Self>) -> bool {
4955 let (a0, a1) = self.split_mask8x32(a);
4956 self.all_true_mask8x16(a0) && self.all_true_mask8x16(a1)
4957 }
4958 #[inline(always)]
4959 fn any_false_mask8x32(self, a: mask8x32<Self>) -> bool {
4960 let (a0, a1) = self.split_mask8x32(a);
4961 self.any_false_mask8x16(a0) || self.any_false_mask8x16(a1)
4962 }
4963 #[inline(always)]
4964 fn all_false_mask8x32(self, a: mask8x32<Self>) -> bool {
4965 let (a0, a1) = self.split_mask8x32(a);
4966 self.all_false_mask8x16(a0) && self.all_false_mask8x16(a1)
4967 }
4968 #[inline(always)]
4969 fn combine_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x64<Self> {
4970 mask8x64 {
4971 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
4972 simd: self,
4973 }
4974 }
4975 #[inline(always)]
4976 fn split_mask8x32(self, a: mask8x32<Self>) -> (mask8x16<Self>, mask8x16<Self>) {
4977 (
4978 mask8x16 {
4979 val: crate::support::Aligned128(a.val.0[0]),
4980 simd: self,
4981 },
4982 mask8x16 {
4983 val: crate::support::Aligned128(a.val.0[1]),
4984 simd: self,
4985 },
4986 )
4987 }
4988 #[inline(always)]
4989 fn splat_i16x16(self, val: i16) -> i16x16<Self> {
4990 let half = self.splat_i16x8(val);
4991 self.combine_i16x8(half, half)
4992 }
4993 #[inline(always)]
4994 fn load_array_i16x16(self, val: [i16; 16usize]) -> i16x16<Self> {
4995 i16x16 {
4996 val: crate::transmute::checked_transmute_copy(&val),
4997 simd: self,
4998 }
4999 }
5000 #[inline(always)]
5001 fn load_array_ref_i16x16(self, val: &[i16; 16usize]) -> i16x16<Self> {
5002 i16x16 {
5003 val: crate::transmute::checked_transmute_copy(val),
5004 simd: self,
5005 }
5006 }
5007 #[inline(always)]
5008 fn as_array_i16x16(self, a: i16x16<Self>) -> [i16; 16usize] {
5009 crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i16; 16usize]>(&a.val.0)
5010 }
5011 #[inline(always)]
5012 fn as_array_ref_i16x16(self, a: &i16x16<Self>) -> &[i16; 16usize] {
5013 crate::transmute::checked_cast_ref::<[__m128i; 2usize], [i16; 16usize]>(&a.val.0)
5014 }
5015 #[inline(always)]
5016 fn as_array_mut_i16x16(self, a: &mut i16x16<Self>) -> &mut [i16; 16usize] {
5017 crate::transmute::checked_cast_mut::<[__m128i; 2usize], [i16; 16usize]>(&mut a.val.0)
5018 }
5019 #[inline(always)]
5020 fn store_array_i16x16(self, a: i16x16<Self>, dest: &mut [i16; 16usize]) -> () {
5021 crate::transmute::checked_transmute_store(a.val.0, dest);
5022 }
5023 #[inline(always)]
5024 fn cvt_from_bytes_i16x16(self, a: u8x32<Self>) -> i16x16<Self> {
5025 i16x16 {
5026 val: crate::transmute::checked_transmute_copy(&a.val),
5027 simd: self,
5028 }
5029 }
5030 #[inline(always)]
5031 fn cvt_to_bytes_i16x16(self, a: i16x16<Self>) -> u8x32<Self> {
5032 u8x32 {
5033 val: crate::transmute::checked_transmute_copy(&a.val),
5034 simd: self,
5035 }
5036 }
5037 #[inline(always)]
5038 fn slide_i16x16<const SHIFT: usize>(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5039 if SHIFT >= 16usize {
5040 return b;
5041 }
5042 let result = cross_block_alignr_128x2(
5043 self,
5044 self.cvt_to_bytes_i16x16(b).val.0,
5045 self.cvt_to_bytes_i16x16(a).val.0,
5046 SHIFT * 2usize,
5047 );
5048 self.cvt_from_bytes_i16x16(u8x32 {
5049 val: crate::support::Aligned256(result),
5050 simd: self,
5051 })
5052 }
5053 #[inline(always)]
5054 fn slide_within_blocks_i16x16<const SHIFT: usize>(
5055 self,
5056 a: i16x16<Self>,
5057 b: i16x16<Self>,
5058 ) -> i16x16<Self> {
5059 let (a0, a1) = self.split_i16x16(a);
5060 let (b0, b1) = self.split_i16x16(b);
5061 self.combine_i16x8(
5062 self.slide_within_blocks_i16x8::<SHIFT>(a0, b0),
5063 self.slide_within_blocks_i16x8::<SHIFT>(a1, b1),
5064 )
5065 }
5066 #[inline(always)]
5067 fn swizzle_dyn_within_blocks_i16x16(
5068 self,
5069 a: i16x16<Self>,
5070 indices: u8x32<Self>,
5071 ) -> i16x16<Self> {
5072 let (a0, a1) = self.split_i16x16(a);
5073 let (indices0, indices1) = self.split_u8x32(indices);
5074 self.combine_i16x8(
5075 self.swizzle_dyn_within_blocks_i16x8(a0, indices0),
5076 self.swizzle_dyn_within_blocks_i16x8(a1, indices1),
5077 )
5078 }
5079 #[inline(always)]
5080 fn add_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5081 let (a0, a1) = self.split_i16x16(a);
5082 let (b0, b1) = self.split_i16x16(b);
5083 self.combine_i16x8(self.add_i16x8(a0, b0), self.add_i16x8(a1, b1))
5084 }
5085 #[inline(always)]
5086 fn sub_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5087 let (a0, a1) = self.split_i16x16(a);
5088 let (b0, b1) = self.split_i16x16(b);
5089 self.combine_i16x8(self.sub_i16x8(a0, b0), self.sub_i16x8(a1, b1))
5090 }
5091 #[inline(always)]
5092 fn mul_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5093 let (a0, a1) = self.split_i16x16(a);
5094 let (b0, b1) = self.split_i16x16(b);
5095 self.combine_i16x8(self.mul_i16x8(a0, b0), self.mul_i16x8(a1, b1))
5096 }
5097 #[inline(always)]
5098 fn and_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5099 let (a0, a1) = self.split_i16x16(a);
5100 let (b0, b1) = self.split_i16x16(b);
5101 self.combine_i16x8(self.and_i16x8(a0, b0), self.and_i16x8(a1, b1))
5102 }
5103 #[inline(always)]
5104 fn or_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5105 let (a0, a1) = self.split_i16x16(a);
5106 let (b0, b1) = self.split_i16x16(b);
5107 self.combine_i16x8(self.or_i16x8(a0, b0), self.or_i16x8(a1, b1))
5108 }
5109 #[inline(always)]
5110 fn xor_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5111 let (a0, a1) = self.split_i16x16(a);
5112 let (b0, b1) = self.split_i16x16(b);
5113 self.combine_i16x8(self.xor_i16x8(a0, b0), self.xor_i16x8(a1, b1))
5114 }
5115 #[inline(always)]
5116 fn not_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
5117 let (a0, a1) = self.split_i16x16(a);
5118 self.combine_i16x8(self.not_i16x8(a0), self.not_i16x8(a1))
5119 }
5120 #[inline(always)]
5121 fn shl_i16x16(self, a: i16x16<Self>, shift: u32) -> i16x16<Self> {
5122 let (a0, a1) = self.split_i16x16(a);
5123 self.combine_i16x8(self.shl_i16x8(a0, shift), self.shl_i16x8(a1, shift))
5124 }
5125 #[inline(always)]
5126 fn shlv_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5127 let (a0, a1) = self.split_i16x16(a);
5128 let (b0, b1) = self.split_i16x16(b);
5129 self.combine_i16x8(self.shlv_i16x8(a0, b0), self.shlv_i16x8(a1, b1))
5130 }
5131 #[inline(always)]
5132 fn shr_i16x16(self, a: i16x16<Self>, shift: u32) -> i16x16<Self> {
5133 let (a0, a1) = self.split_i16x16(a);
5134 self.combine_i16x8(self.shr_i16x8(a0, shift), self.shr_i16x8(a1, shift))
5135 }
5136 #[inline(always)]
5137 fn shrv_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5138 let (a0, a1) = self.split_i16x16(a);
5139 let (b0, b1) = self.split_i16x16(b);
5140 self.combine_i16x8(self.shrv_i16x8(a0, b0), self.shrv_i16x8(a1, b1))
5141 }
5142 #[inline(always)]
5143 fn simd_eq_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
5144 let (a0, a1) = self.split_i16x16(a);
5145 let (b0, b1) = self.split_i16x16(b);
5146 self.combine_mask16x8(self.simd_eq_i16x8(a0, b0), self.simd_eq_i16x8(a1, b1))
5147 }
5148 #[inline(always)]
5149 fn simd_lt_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
5150 let (a0, a1) = self.split_i16x16(a);
5151 let (b0, b1) = self.split_i16x16(b);
5152 self.combine_mask16x8(self.simd_lt_i16x8(a0, b0), self.simd_lt_i16x8(a1, b1))
5153 }
5154 #[inline(always)]
5155 fn simd_le_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
5156 let (a0, a1) = self.split_i16x16(a);
5157 let (b0, b1) = self.split_i16x16(b);
5158 self.combine_mask16x8(self.simd_le_i16x8(a0, b0), self.simd_le_i16x8(a1, b1))
5159 }
5160 #[inline(always)]
5161 fn simd_ge_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
5162 let (a0, a1) = self.split_i16x16(a);
5163 let (b0, b1) = self.split_i16x16(b);
5164 self.combine_mask16x8(self.simd_ge_i16x8(a0, b0), self.simd_ge_i16x8(a1, b1))
5165 }
5166 #[inline(always)]
5167 fn simd_gt_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
5168 let (a0, a1) = self.split_i16x16(a);
5169 let (b0, b1) = self.split_i16x16(b);
5170 self.combine_mask16x8(self.simd_gt_i16x8(a0, b0), self.simd_gt_i16x8(a1, b1))
5171 }
5172 #[inline(always)]
5173 fn zip_low_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5174 let (a0, _) = self.split_i16x16(a);
5175 let (b0, _) = self.split_i16x16(b);
5176 self.combine_i16x8(self.zip_low_i16x8(a0, b0), self.zip_high_i16x8(a0, b0))
5177 }
5178 #[inline(always)]
5179 fn zip_high_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5180 let (_, a1) = self.split_i16x16(a);
5181 let (_, b1) = self.split_i16x16(b);
5182 self.combine_i16x8(self.zip_low_i16x8(a1, b1), self.zip_high_i16x8(a1, b1))
5183 }
5184 #[inline(always)]
5185 fn unzip_low_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5186 let (a0, a1) = self.split_i16x16(a);
5187 let (b0, b1) = self.split_i16x16(b);
5188 self.combine_i16x8(self.unzip_low_i16x8(a0, a1), self.unzip_low_i16x8(b0, b1))
5189 }
5190 #[inline(always)]
5191 fn unzip_high_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5192 let (a0, a1) = self.split_i16x16(a);
5193 let (b0, b1) = self.split_i16x16(b);
5194 self.combine_i16x8(self.unzip_high_i16x8(a0, a1), self.unzip_high_i16x8(b0, b1))
5195 }
5196 #[inline(always)]
5197 fn interleave_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i16x16<Self>, i16x16<Self>) {
5198 let (a0, a1) = self.split_i16x16(a);
5199 let (b0, b1) = self.split_i16x16(b);
5200 let lo_lo = self.zip_low_i16x8(a0, b0);
5201 let lo_hi = self.zip_high_i16x8(a0, b0);
5202 let hi_lo = self.zip_low_i16x8(a1, b1);
5203 let hi_hi = self.zip_high_i16x8(a1, b1);
5204 (
5205 self.combine_i16x8(lo_lo, lo_hi),
5206 self.combine_i16x8(hi_lo, hi_hi),
5207 )
5208 }
5209 #[inline(always)]
5210 fn deinterleave_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i16x16<Self>, i16x16<Self>) {
5211 let (a0, a1) = self.split_i16x16(a);
5212 let (b0, b1) = self.split_i16x16(b);
5213 let lo_even = self.unzip_low_i16x8(a0, a1);
5214 let lo_odd = self.unzip_high_i16x8(a0, a1);
5215 let hi_even = self.unzip_low_i16x8(b0, b1);
5216 let hi_odd = self.unzip_high_i16x8(b0, b1);
5217 (
5218 self.combine_i16x8(lo_even, hi_even),
5219 self.combine_i16x8(lo_odd, hi_odd),
5220 )
5221 }
5222 #[inline(always)]
5223 fn select_i16x16(self, a: mask16x16<Self>, b: i16x16<Self>, c: i16x16<Self>) -> i16x16<Self> {
5224 let (a0, a1) = self.split_mask16x16(a);
5225 let (b0, b1) = self.split_i16x16(b);
5226 let (c0, c1) = self.split_i16x16(c);
5227 self.combine_i16x8(self.select_i16x8(a0, b0, c0), self.select_i16x8(a1, b1, c1))
5228 }
5229 #[inline(always)]
5230 fn min_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5231 let (a0, a1) = self.split_i16x16(a);
5232 let (b0, b1) = self.split_i16x16(b);
5233 self.combine_i16x8(self.min_i16x8(a0, b0), self.min_i16x8(a1, b1))
5234 }
5235 #[inline(always)]
5236 fn max_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
5237 let (a0, a1) = self.split_i16x16(a);
5238 let (b0, b1) = self.split_i16x16(b);
5239 self.combine_i16x8(self.max_i16x8(a0, b0), self.max_i16x8(a1, b1))
5240 }
5241 #[inline(always)]
5242 fn combine_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x32<Self> {
5243 i16x32 {
5244 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
5245 simd: self,
5246 }
5247 }
5248 #[inline(always)]
5249 fn split_i16x16(self, a: i16x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
5250 (
5251 i16x8 {
5252 val: crate::support::Aligned128(a.val.0[0]),
5253 simd: self,
5254 },
5255 i16x8 {
5256 val: crate::support::Aligned128(a.val.0[1]),
5257 simd: self,
5258 },
5259 )
5260 }
5261 #[inline(always)]
5262 fn neg_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
5263 let (a0, a1) = self.split_i16x16(a);
5264 self.combine_i16x8(self.neg_i16x8(a0), self.neg_i16x8(a1))
5265 }
5266 #[inline(always)]
5267 fn reinterpret_u8_i16x16(self, a: i16x16<Self>) -> u8x32<Self> {
5268 let (a0, a1) = self.split_i16x16(a);
5269 self.combine_u8x16(self.reinterpret_u8_i16x8(a0), self.reinterpret_u8_i16x8(a1))
5270 }
5271 #[inline(always)]
5272 fn reinterpret_u32_i16x16(self, a: i16x16<Self>) -> u32x8<Self> {
5273 let (a0, a1) = self.split_i16x16(a);
5274 self.combine_u32x4(
5275 self.reinterpret_u32_i16x8(a0),
5276 self.reinterpret_u32_i16x8(a1),
5277 )
5278 }
5279 #[inline(always)]
5280 fn splat_u16x16(self, val: u16) -> u16x16<Self> {
5281 let half = self.splat_u16x8(val);
5282 self.combine_u16x8(half, half)
5283 }
5284 #[inline(always)]
5285 fn load_array_u16x16(self, val: [u16; 16usize]) -> u16x16<Self> {
5286 u16x16 {
5287 val: crate::transmute::checked_transmute_copy(&val),
5288 simd: self,
5289 }
5290 }
5291 #[inline(always)]
5292 fn load_array_ref_u16x16(self, val: &[u16; 16usize]) -> u16x16<Self> {
5293 u16x16 {
5294 val: crate::transmute::checked_transmute_copy(val),
5295 simd: self,
5296 }
5297 }
5298 #[inline(always)]
5299 fn as_array_u16x16(self, a: u16x16<Self>) -> [u16; 16usize] {
5300 crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [u16; 16usize]>(&a.val.0)
5301 }
5302 #[inline(always)]
5303 fn as_array_ref_u16x16(self, a: &u16x16<Self>) -> &[u16; 16usize] {
5304 crate::transmute::checked_cast_ref::<[__m128i; 2usize], [u16; 16usize]>(&a.val.0)
5305 }
5306 #[inline(always)]
5307 fn as_array_mut_u16x16(self, a: &mut u16x16<Self>) -> &mut [u16; 16usize] {
5308 crate::transmute::checked_cast_mut::<[__m128i; 2usize], [u16; 16usize]>(&mut a.val.0)
5309 }
5310 #[inline(always)]
5311 fn store_array_u16x16(self, a: u16x16<Self>, dest: &mut [u16; 16usize]) -> () {
5312 crate::transmute::checked_transmute_store(a.val.0, dest);
5313 }
5314 #[inline(always)]
5315 fn cvt_from_bytes_u16x16(self, a: u8x32<Self>) -> u16x16<Self> {
5316 u16x16 {
5317 val: crate::transmute::checked_transmute_copy(&a.val),
5318 simd: self,
5319 }
5320 }
5321 #[inline(always)]
5322 fn cvt_to_bytes_u16x16(self, a: u16x16<Self>) -> u8x32<Self> {
5323 u8x32 {
5324 val: crate::transmute::checked_transmute_copy(&a.val),
5325 simd: self,
5326 }
5327 }
5328 #[inline(always)]
5329 fn slide_u16x16<const SHIFT: usize>(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5330 if SHIFT >= 16usize {
5331 return b;
5332 }
5333 let result = cross_block_alignr_128x2(
5334 self,
5335 self.cvt_to_bytes_u16x16(b).val.0,
5336 self.cvt_to_bytes_u16x16(a).val.0,
5337 SHIFT * 2usize,
5338 );
5339 self.cvt_from_bytes_u16x16(u8x32 {
5340 val: crate::support::Aligned256(result),
5341 simd: self,
5342 })
5343 }
5344 #[inline(always)]
5345 fn slide_within_blocks_u16x16<const SHIFT: usize>(
5346 self,
5347 a: u16x16<Self>,
5348 b: u16x16<Self>,
5349 ) -> u16x16<Self> {
5350 let (a0, a1) = self.split_u16x16(a);
5351 let (b0, b1) = self.split_u16x16(b);
5352 self.combine_u16x8(
5353 self.slide_within_blocks_u16x8::<SHIFT>(a0, b0),
5354 self.slide_within_blocks_u16x8::<SHIFT>(a1, b1),
5355 )
5356 }
5357 #[inline(always)]
5358 fn swizzle_dyn_within_blocks_u16x16(
5359 self,
5360 a: u16x16<Self>,
5361 indices: u8x32<Self>,
5362 ) -> u16x16<Self> {
5363 let (a0, a1) = self.split_u16x16(a);
5364 let (indices0, indices1) = self.split_u8x32(indices);
5365 self.combine_u16x8(
5366 self.swizzle_dyn_within_blocks_u16x8(a0, indices0),
5367 self.swizzle_dyn_within_blocks_u16x8(a1, indices1),
5368 )
5369 }
5370 #[inline(always)]
5371 fn add_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5372 let (a0, a1) = self.split_u16x16(a);
5373 let (b0, b1) = self.split_u16x16(b);
5374 self.combine_u16x8(self.add_u16x8(a0, b0), self.add_u16x8(a1, b1))
5375 }
5376 #[inline(always)]
5377 fn sub_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5378 let (a0, a1) = self.split_u16x16(a);
5379 let (b0, b1) = self.split_u16x16(b);
5380 self.combine_u16x8(self.sub_u16x8(a0, b0), self.sub_u16x8(a1, b1))
5381 }
5382 #[inline(always)]
5383 fn mul_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5384 let (a0, a1) = self.split_u16x16(a);
5385 let (b0, b1) = self.split_u16x16(b);
5386 self.combine_u16x8(self.mul_u16x8(a0, b0), self.mul_u16x8(a1, b1))
5387 }
5388 #[inline(always)]
5389 fn and_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5390 let (a0, a1) = self.split_u16x16(a);
5391 let (b0, b1) = self.split_u16x16(b);
5392 self.combine_u16x8(self.and_u16x8(a0, b0), self.and_u16x8(a1, b1))
5393 }
5394 #[inline(always)]
5395 fn or_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5396 let (a0, a1) = self.split_u16x16(a);
5397 let (b0, b1) = self.split_u16x16(b);
5398 self.combine_u16x8(self.or_u16x8(a0, b0), self.or_u16x8(a1, b1))
5399 }
5400 #[inline(always)]
5401 fn xor_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5402 let (a0, a1) = self.split_u16x16(a);
5403 let (b0, b1) = self.split_u16x16(b);
5404 self.combine_u16x8(self.xor_u16x8(a0, b0), self.xor_u16x8(a1, b1))
5405 }
5406 #[inline(always)]
5407 fn not_u16x16(self, a: u16x16<Self>) -> u16x16<Self> {
5408 let (a0, a1) = self.split_u16x16(a);
5409 self.combine_u16x8(self.not_u16x8(a0), self.not_u16x8(a1))
5410 }
5411 #[inline(always)]
5412 fn shl_u16x16(self, a: u16x16<Self>, shift: u32) -> u16x16<Self> {
5413 let (a0, a1) = self.split_u16x16(a);
5414 self.combine_u16x8(self.shl_u16x8(a0, shift), self.shl_u16x8(a1, shift))
5415 }
5416 #[inline(always)]
5417 fn shlv_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5418 let (a0, a1) = self.split_u16x16(a);
5419 let (b0, b1) = self.split_u16x16(b);
5420 self.combine_u16x8(self.shlv_u16x8(a0, b0), self.shlv_u16x8(a1, b1))
5421 }
5422 #[inline(always)]
5423 fn shr_u16x16(self, a: u16x16<Self>, shift: u32) -> u16x16<Self> {
5424 let (a0, a1) = self.split_u16x16(a);
5425 self.combine_u16x8(self.shr_u16x8(a0, shift), self.shr_u16x8(a1, shift))
5426 }
5427 #[inline(always)]
5428 fn shrv_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5429 let (a0, a1) = self.split_u16x16(a);
5430 let (b0, b1) = self.split_u16x16(b);
5431 self.combine_u16x8(self.shrv_u16x8(a0, b0), self.shrv_u16x8(a1, b1))
5432 }
5433 #[inline(always)]
5434 fn simd_eq_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
5435 let (a0, a1) = self.split_u16x16(a);
5436 let (b0, b1) = self.split_u16x16(b);
5437 self.combine_mask16x8(self.simd_eq_u16x8(a0, b0), self.simd_eq_u16x8(a1, b1))
5438 }
5439 #[inline(always)]
5440 fn simd_lt_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
5441 let (a0, a1) = self.split_u16x16(a);
5442 let (b0, b1) = self.split_u16x16(b);
5443 self.combine_mask16x8(self.simd_lt_u16x8(a0, b0), self.simd_lt_u16x8(a1, b1))
5444 }
5445 #[inline(always)]
5446 fn simd_le_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
5447 let (a0, a1) = self.split_u16x16(a);
5448 let (b0, b1) = self.split_u16x16(b);
5449 self.combine_mask16x8(self.simd_le_u16x8(a0, b0), self.simd_le_u16x8(a1, b1))
5450 }
5451 #[inline(always)]
5452 fn simd_ge_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
5453 let (a0, a1) = self.split_u16x16(a);
5454 let (b0, b1) = self.split_u16x16(b);
5455 self.combine_mask16x8(self.simd_ge_u16x8(a0, b0), self.simd_ge_u16x8(a1, b1))
5456 }
5457 #[inline(always)]
5458 fn simd_gt_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
5459 let (a0, a1) = self.split_u16x16(a);
5460 let (b0, b1) = self.split_u16x16(b);
5461 self.combine_mask16x8(self.simd_gt_u16x8(a0, b0), self.simd_gt_u16x8(a1, b1))
5462 }
5463 #[inline(always)]
5464 fn zip_low_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5465 let (a0, _) = self.split_u16x16(a);
5466 let (b0, _) = self.split_u16x16(b);
5467 self.combine_u16x8(self.zip_low_u16x8(a0, b0), self.zip_high_u16x8(a0, b0))
5468 }
5469 #[inline(always)]
5470 fn zip_high_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5471 let (_, a1) = self.split_u16x16(a);
5472 let (_, b1) = self.split_u16x16(b);
5473 self.combine_u16x8(self.zip_low_u16x8(a1, b1), self.zip_high_u16x8(a1, b1))
5474 }
5475 #[inline(always)]
5476 fn unzip_low_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5477 let (a0, a1) = self.split_u16x16(a);
5478 let (b0, b1) = self.split_u16x16(b);
5479 self.combine_u16x8(self.unzip_low_u16x8(a0, a1), self.unzip_low_u16x8(b0, b1))
5480 }
5481 #[inline(always)]
5482 fn unzip_high_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5483 let (a0, a1) = self.split_u16x16(a);
5484 let (b0, b1) = self.split_u16x16(b);
5485 self.combine_u16x8(self.unzip_high_u16x8(a0, a1), self.unzip_high_u16x8(b0, b1))
5486 }
5487 #[inline(always)]
5488 fn interleave_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u16x16<Self>, u16x16<Self>) {
5489 let (a0, a1) = self.split_u16x16(a);
5490 let (b0, b1) = self.split_u16x16(b);
5491 let lo_lo = self.zip_low_u16x8(a0, b0);
5492 let lo_hi = self.zip_high_u16x8(a0, b0);
5493 let hi_lo = self.zip_low_u16x8(a1, b1);
5494 let hi_hi = self.zip_high_u16x8(a1, b1);
5495 (
5496 self.combine_u16x8(lo_lo, lo_hi),
5497 self.combine_u16x8(hi_lo, hi_hi),
5498 )
5499 }
5500 #[inline(always)]
5501 fn deinterleave_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u16x16<Self>, u16x16<Self>) {
5502 let (a0, a1) = self.split_u16x16(a);
5503 let (b0, b1) = self.split_u16x16(b);
5504 let lo_even = self.unzip_low_u16x8(a0, a1);
5505 let lo_odd = self.unzip_high_u16x8(a0, a1);
5506 let hi_even = self.unzip_low_u16x8(b0, b1);
5507 let hi_odd = self.unzip_high_u16x8(b0, b1);
5508 (
5509 self.combine_u16x8(lo_even, hi_even),
5510 self.combine_u16x8(lo_odd, hi_odd),
5511 )
5512 }
5513 #[inline(always)]
5514 fn select_u16x16(self, a: mask16x16<Self>, b: u16x16<Self>, c: u16x16<Self>) -> u16x16<Self> {
5515 let (a0, a1) = self.split_mask16x16(a);
5516 let (b0, b1) = self.split_u16x16(b);
5517 let (c0, c1) = self.split_u16x16(c);
5518 self.combine_u16x8(self.select_u16x8(a0, b0, c0), self.select_u16x8(a1, b1, c1))
5519 }
5520 #[inline(always)]
5521 fn min_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5522 let (a0, a1) = self.split_u16x16(a);
5523 let (b0, b1) = self.split_u16x16(b);
5524 self.combine_u16x8(self.min_u16x8(a0, b0), self.min_u16x8(a1, b1))
5525 }
5526 #[inline(always)]
5527 fn max_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
5528 let (a0, a1) = self.split_u16x16(a);
5529 let (b0, b1) = self.split_u16x16(b);
5530 self.combine_u16x8(self.max_u16x8(a0, b0), self.max_u16x8(a1, b1))
5531 }
5532 #[inline(always)]
5533 fn combine_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x32<Self> {
5534 u16x32 {
5535 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
5536 simd: self,
5537 }
5538 }
5539 #[inline(always)]
5540 fn split_u16x16(self, a: u16x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
5541 (
5542 u16x8 {
5543 val: crate::support::Aligned128(a.val.0[0]),
5544 simd: self,
5545 },
5546 u16x8 {
5547 val: crate::support::Aligned128(a.val.0[1]),
5548 simd: self,
5549 },
5550 )
5551 }
5552 #[inline(always)]
5553 fn narrow_u16x16(self, a: u16x16<Self>) -> u8x16<Self> {
5554 crate::kernel!(
5555 #[inline(always)]
5556 fn kernel(token: Sse4_2, a: u16x16<Sse4_2>) -> u8x16<Sse4_2> {
5557 let (a, b) = token.split_u16x16(a);
5558 let mask = _mm_set1_epi16(0xFF);
5559 let lo_masked = _mm_and_si128(a.into(), mask);
5560 let hi_masked = _mm_and_si128(b.into(), mask);
5561 let result = _mm_packus_epi16(lo_masked, hi_masked);
5562 result.simd_into(token)
5563 }
5564 );
5565 kernel(self, a)
5566 }
5567 #[inline(always)]
5568 fn reinterpret_u8_u16x16(self, a: u16x16<Self>) -> u8x32<Self> {
5569 let (a0, a1) = self.split_u16x16(a);
5570 self.combine_u8x16(self.reinterpret_u8_u16x8(a0), self.reinterpret_u8_u16x8(a1))
5571 }
5572 #[inline(always)]
5573 fn reinterpret_u32_u16x16(self, a: u16x16<Self>) -> u32x8<Self> {
5574 let (a0, a1) = self.split_u16x16(a);
5575 self.combine_u32x4(
5576 self.reinterpret_u32_u16x8(a0),
5577 self.reinterpret_u32_u16x8(a1),
5578 )
5579 }
5580 #[inline(always)]
5581 fn splat_mask16x16(self, val: bool) -> mask16x16<Self> {
5582 let half = self.splat_mask16x8(val);
5583 self.combine_mask16x8(half, half)
5584 }
5585 #[inline(always)]
5586 fn load_array_mask16x16(self, val: [i16; 16usize]) -> mask16x16<Self> {
5587 mask16x16 {
5588 val: crate::transmute::checked_transmute_copy(&val),
5589 simd: self,
5590 }
5591 }
5592 #[inline(always)]
5593 fn as_array_mask16x16(self, a: mask16x16<Self>) -> [i16; 16usize] {
5594 crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i16; 16usize]>(&a.val.0)
5595 }
5596 #[inline(always)]
5597 fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16<Self> {
5598 let lo = self.from_bitmask_mask16x8(bits);
5599 let hi = self.from_bitmask_mask16x8(bits >> 8usize);
5600 self.combine_mask16x8(lo, hi)
5601 }
5602 #[inline(always)]
5603 fn to_bitmask_mask16x16(self, a: mask16x16<Self>) -> u64 {
5604 crate::kernel!(
5605 #[inline(always)]
5606 fn kernel(token: Sse4_2, a: mask16x16<Sse4_2>) -> u64 {
5607 {
5608 let packed = _mm_packs_epi16(a.val.0[0], a.val.0[1]);
5609 _mm_movemask_epi8(packed) as u32 as u64
5610 }
5611 }
5612 );
5613 kernel(self, a)
5614 }
5615 #[inline(always)]
5616 fn set_mask16x16(self, a: &mut mask16x16<Self>, index: usize, value: bool) -> () {
5617 assert!(
5618 index < 16usize,
5619 "mask lane index {index} is out of bounds for {} lanes",
5620 16usize
5621 );
5622 let mut lanes = self.as_array_mask16x16(*a);
5623 lanes[index] = if value { !0 } else { 0 };
5624 *a = self.load_array_mask16x16(lanes);
5625 }
5626 #[inline(always)]
5627 fn and_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
5628 let (a0, a1) = self.split_mask16x16(a);
5629 let (b0, b1) = self.split_mask16x16(b);
5630 self.combine_mask16x8(self.and_mask16x8(a0, b0), self.and_mask16x8(a1, b1))
5631 }
5632 #[inline(always)]
5633 fn or_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
5634 let (a0, a1) = self.split_mask16x16(a);
5635 let (b0, b1) = self.split_mask16x16(b);
5636 self.combine_mask16x8(self.or_mask16x8(a0, b0), self.or_mask16x8(a1, b1))
5637 }
5638 #[inline(always)]
5639 fn xor_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
5640 let (a0, a1) = self.split_mask16x16(a);
5641 let (b0, b1) = self.split_mask16x16(b);
5642 self.combine_mask16x8(self.xor_mask16x8(a0, b0), self.xor_mask16x8(a1, b1))
5643 }
5644 #[inline(always)]
5645 fn not_mask16x16(self, a: mask16x16<Self>) -> mask16x16<Self> {
5646 let (a0, a1) = self.split_mask16x16(a);
5647 self.combine_mask16x8(self.not_mask16x8(a0), self.not_mask16x8(a1))
5648 }
5649 #[inline(always)]
5650 fn select_mask16x16(
5651 self,
5652 a: mask16x16<Self>,
5653 b: mask16x16<Self>,
5654 c: mask16x16<Self>,
5655 ) -> mask16x16<Self> {
5656 let (a0, a1) = self.split_mask16x16(a);
5657 let (b0, b1) = self.split_mask16x16(b);
5658 let (c0, c1) = self.split_mask16x16(c);
5659 self.combine_mask16x8(
5660 self.select_mask16x8(a0, b0, c0),
5661 self.select_mask16x8(a1, b1, c1),
5662 )
5663 }
5664 #[inline(always)]
5665 fn simd_eq_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
5666 let (a0, a1) = self.split_mask16x16(a);
5667 let (b0, b1) = self.split_mask16x16(b);
5668 self.combine_mask16x8(self.simd_eq_mask16x8(a0, b0), self.simd_eq_mask16x8(a1, b1))
5669 }
5670 #[inline(always)]
5671 fn any_true_mask16x16(self, a: mask16x16<Self>) -> bool {
5672 let (a0, a1) = self.split_mask16x16(a);
5673 self.any_true_mask16x8(a0) || self.any_true_mask16x8(a1)
5674 }
5675 #[inline(always)]
5676 fn all_true_mask16x16(self, a: mask16x16<Self>) -> bool {
5677 let (a0, a1) = self.split_mask16x16(a);
5678 self.all_true_mask16x8(a0) && self.all_true_mask16x8(a1)
5679 }
5680 #[inline(always)]
5681 fn any_false_mask16x16(self, a: mask16x16<Self>) -> bool {
5682 let (a0, a1) = self.split_mask16x16(a);
5683 self.any_false_mask16x8(a0) || self.any_false_mask16x8(a1)
5684 }
5685 #[inline(always)]
5686 fn all_false_mask16x16(self, a: mask16x16<Self>) -> bool {
5687 let (a0, a1) = self.split_mask16x16(a);
5688 self.all_false_mask16x8(a0) && self.all_false_mask16x8(a1)
5689 }
5690 #[inline(always)]
5691 fn combine_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x32<Self> {
5692 mask16x32 {
5693 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
5694 simd: self,
5695 }
5696 }
5697 #[inline(always)]
5698 fn split_mask16x16(self, a: mask16x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
5699 (
5700 mask16x8 {
5701 val: crate::support::Aligned128(a.val.0[0]),
5702 simd: self,
5703 },
5704 mask16x8 {
5705 val: crate::support::Aligned128(a.val.0[1]),
5706 simd: self,
5707 },
5708 )
5709 }
5710 #[inline(always)]
5711 fn splat_i32x8(self, val: i32) -> i32x8<Self> {
5712 let half = self.splat_i32x4(val);
5713 self.combine_i32x4(half, half)
5714 }
5715 #[inline(always)]
5716 fn load_array_i32x8(self, val: [i32; 8usize]) -> i32x8<Self> {
5717 i32x8 {
5718 val: crate::transmute::checked_transmute_copy(&val),
5719 simd: self,
5720 }
5721 }
5722 #[inline(always)]
5723 fn load_array_ref_i32x8(self, val: &[i32; 8usize]) -> i32x8<Self> {
5724 i32x8 {
5725 val: crate::transmute::checked_transmute_copy(val),
5726 simd: self,
5727 }
5728 }
5729 #[inline(always)]
5730 fn as_array_i32x8(self, a: i32x8<Self>) -> [i32; 8usize] {
5731 crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i32; 8usize]>(&a.val.0)
5732 }
5733 #[inline(always)]
5734 fn as_array_ref_i32x8(self, a: &i32x8<Self>) -> &[i32; 8usize] {
5735 crate::transmute::checked_cast_ref::<[__m128i; 2usize], [i32; 8usize]>(&a.val.0)
5736 }
5737 #[inline(always)]
5738 fn as_array_mut_i32x8(self, a: &mut i32x8<Self>) -> &mut [i32; 8usize] {
5739 crate::transmute::checked_cast_mut::<[__m128i; 2usize], [i32; 8usize]>(&mut a.val.0)
5740 }
5741 #[inline(always)]
5742 fn store_array_i32x8(self, a: i32x8<Self>, dest: &mut [i32; 8usize]) -> () {
5743 crate::transmute::checked_transmute_store(a.val.0, dest);
5744 }
5745 #[inline(always)]
5746 fn cvt_from_bytes_i32x8(self, a: u8x32<Self>) -> i32x8<Self> {
5747 i32x8 {
5748 val: crate::transmute::checked_transmute_copy(&a.val),
5749 simd: self,
5750 }
5751 }
5752 #[inline(always)]
5753 fn cvt_to_bytes_i32x8(self, a: i32x8<Self>) -> u8x32<Self> {
5754 u8x32 {
5755 val: crate::transmute::checked_transmute_copy(&a.val),
5756 simd: self,
5757 }
5758 }
5759 #[inline(always)]
5760 fn slide_i32x8<const SHIFT: usize>(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5761 if SHIFT >= 8usize {
5762 return b;
5763 }
5764 let result = cross_block_alignr_128x2(
5765 self,
5766 self.cvt_to_bytes_i32x8(b).val.0,
5767 self.cvt_to_bytes_i32x8(a).val.0,
5768 SHIFT * 4usize,
5769 );
5770 self.cvt_from_bytes_i32x8(u8x32 {
5771 val: crate::support::Aligned256(result),
5772 simd: self,
5773 })
5774 }
5775 #[inline(always)]
5776 fn slide_within_blocks_i32x8<const SHIFT: usize>(
5777 self,
5778 a: i32x8<Self>,
5779 b: i32x8<Self>,
5780 ) -> i32x8<Self> {
5781 let (a0, a1) = self.split_i32x8(a);
5782 let (b0, b1) = self.split_i32x8(b);
5783 self.combine_i32x4(
5784 self.slide_within_blocks_i32x4::<SHIFT>(a0, b0),
5785 self.slide_within_blocks_i32x4::<SHIFT>(a1, b1),
5786 )
5787 }
5788 #[inline(always)]
5789 fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8<Self>, indices: u8x32<Self>) -> i32x8<Self> {
5790 let (a0, a1) = self.split_i32x8(a);
5791 let (indices0, indices1) = self.split_u8x32(indices);
5792 self.combine_i32x4(
5793 self.swizzle_dyn_within_blocks_i32x4(a0, indices0),
5794 self.swizzle_dyn_within_blocks_i32x4(a1, indices1),
5795 )
5796 }
5797 #[inline(always)]
5798 fn add_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5799 let (a0, a1) = self.split_i32x8(a);
5800 let (b0, b1) = self.split_i32x8(b);
5801 self.combine_i32x4(self.add_i32x4(a0, b0), self.add_i32x4(a1, b1))
5802 }
5803 #[inline(always)]
5804 fn sub_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5805 let (a0, a1) = self.split_i32x8(a);
5806 let (b0, b1) = self.split_i32x8(b);
5807 self.combine_i32x4(self.sub_i32x4(a0, b0), self.sub_i32x4(a1, b1))
5808 }
5809 #[inline(always)]
5810 fn mul_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5811 let (a0, a1) = self.split_i32x8(a);
5812 let (b0, b1) = self.split_i32x8(b);
5813 self.combine_i32x4(self.mul_i32x4(a0, b0), self.mul_i32x4(a1, b1))
5814 }
5815 #[inline(always)]
5816 fn and_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5817 let (a0, a1) = self.split_i32x8(a);
5818 let (b0, b1) = self.split_i32x8(b);
5819 self.combine_i32x4(self.and_i32x4(a0, b0), self.and_i32x4(a1, b1))
5820 }
5821 #[inline(always)]
5822 fn or_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5823 let (a0, a1) = self.split_i32x8(a);
5824 let (b0, b1) = self.split_i32x8(b);
5825 self.combine_i32x4(self.or_i32x4(a0, b0), self.or_i32x4(a1, b1))
5826 }
5827 #[inline(always)]
5828 fn xor_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5829 let (a0, a1) = self.split_i32x8(a);
5830 let (b0, b1) = self.split_i32x8(b);
5831 self.combine_i32x4(self.xor_i32x4(a0, b0), self.xor_i32x4(a1, b1))
5832 }
5833 #[inline(always)]
5834 fn not_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
5835 let (a0, a1) = self.split_i32x8(a);
5836 self.combine_i32x4(self.not_i32x4(a0), self.not_i32x4(a1))
5837 }
5838 #[inline(always)]
5839 fn shl_i32x8(self, a: i32x8<Self>, shift: u32) -> i32x8<Self> {
5840 let (a0, a1) = self.split_i32x8(a);
5841 self.combine_i32x4(self.shl_i32x4(a0, shift), self.shl_i32x4(a1, shift))
5842 }
5843 #[inline(always)]
5844 fn shlv_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5845 let (a0, a1) = self.split_i32x8(a);
5846 let (b0, b1) = self.split_i32x8(b);
5847 self.combine_i32x4(self.shlv_i32x4(a0, b0), self.shlv_i32x4(a1, b1))
5848 }
5849 #[inline(always)]
5850 fn shr_i32x8(self, a: i32x8<Self>, shift: u32) -> i32x8<Self> {
5851 let (a0, a1) = self.split_i32x8(a);
5852 self.combine_i32x4(self.shr_i32x4(a0, shift), self.shr_i32x4(a1, shift))
5853 }
5854 #[inline(always)]
5855 fn shrv_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5856 let (a0, a1) = self.split_i32x8(a);
5857 let (b0, b1) = self.split_i32x8(b);
5858 self.combine_i32x4(self.shrv_i32x4(a0, b0), self.shrv_i32x4(a1, b1))
5859 }
5860 #[inline(always)]
5861 fn simd_eq_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
5862 let (a0, a1) = self.split_i32x8(a);
5863 let (b0, b1) = self.split_i32x8(b);
5864 self.combine_mask32x4(self.simd_eq_i32x4(a0, b0), self.simd_eq_i32x4(a1, b1))
5865 }
5866 #[inline(always)]
5867 fn simd_lt_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
5868 let (a0, a1) = self.split_i32x8(a);
5869 let (b0, b1) = self.split_i32x8(b);
5870 self.combine_mask32x4(self.simd_lt_i32x4(a0, b0), self.simd_lt_i32x4(a1, b1))
5871 }
5872 #[inline(always)]
5873 fn simd_le_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
5874 let (a0, a1) = self.split_i32x8(a);
5875 let (b0, b1) = self.split_i32x8(b);
5876 self.combine_mask32x4(self.simd_le_i32x4(a0, b0), self.simd_le_i32x4(a1, b1))
5877 }
5878 #[inline(always)]
5879 fn simd_ge_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
5880 let (a0, a1) = self.split_i32x8(a);
5881 let (b0, b1) = self.split_i32x8(b);
5882 self.combine_mask32x4(self.simd_ge_i32x4(a0, b0), self.simd_ge_i32x4(a1, b1))
5883 }
5884 #[inline(always)]
5885 fn simd_gt_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
5886 let (a0, a1) = self.split_i32x8(a);
5887 let (b0, b1) = self.split_i32x8(b);
5888 self.combine_mask32x4(self.simd_gt_i32x4(a0, b0), self.simd_gt_i32x4(a1, b1))
5889 }
5890 #[inline(always)]
5891 fn zip_low_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5892 let (a0, _) = self.split_i32x8(a);
5893 let (b0, _) = self.split_i32x8(b);
5894 self.combine_i32x4(self.zip_low_i32x4(a0, b0), self.zip_high_i32x4(a0, b0))
5895 }
5896 #[inline(always)]
5897 fn zip_high_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5898 let (_, a1) = self.split_i32x8(a);
5899 let (_, b1) = self.split_i32x8(b);
5900 self.combine_i32x4(self.zip_low_i32x4(a1, b1), self.zip_high_i32x4(a1, b1))
5901 }
5902 #[inline(always)]
5903 fn unzip_low_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5904 let (a0, a1) = self.split_i32x8(a);
5905 let (b0, b1) = self.split_i32x8(b);
5906 self.combine_i32x4(self.unzip_low_i32x4(a0, a1), self.unzip_low_i32x4(b0, b1))
5907 }
5908 #[inline(always)]
5909 fn unzip_high_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5910 let (a0, a1) = self.split_i32x8(a);
5911 let (b0, b1) = self.split_i32x8(b);
5912 self.combine_i32x4(self.unzip_high_i32x4(a0, a1), self.unzip_high_i32x4(b0, b1))
5913 }
5914 #[inline(always)]
5915 fn interleave_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i32x8<Self>, i32x8<Self>) {
5916 let (a0, a1) = self.split_i32x8(a);
5917 let (b0, b1) = self.split_i32x8(b);
5918 let lo_lo = self.zip_low_i32x4(a0, b0);
5919 let lo_hi = self.zip_high_i32x4(a0, b0);
5920 let hi_lo = self.zip_low_i32x4(a1, b1);
5921 let hi_hi = self.zip_high_i32x4(a1, b1);
5922 (
5923 self.combine_i32x4(lo_lo, lo_hi),
5924 self.combine_i32x4(hi_lo, hi_hi),
5925 )
5926 }
5927 #[inline(always)]
5928 fn deinterleave_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i32x8<Self>, i32x8<Self>) {
5929 let (a0, a1) = self.split_i32x8(a);
5930 let (b0, b1) = self.split_i32x8(b);
5931 let lo_even = self.unzip_low_i32x4(a0, a1);
5932 let lo_odd = self.unzip_high_i32x4(a0, a1);
5933 let hi_even = self.unzip_low_i32x4(b0, b1);
5934 let hi_odd = self.unzip_high_i32x4(b0, b1);
5935 (
5936 self.combine_i32x4(lo_even, hi_even),
5937 self.combine_i32x4(lo_odd, hi_odd),
5938 )
5939 }
5940 #[inline(always)]
5941 fn select_i32x8(self, a: mask32x8<Self>, b: i32x8<Self>, c: i32x8<Self>) -> i32x8<Self> {
5942 let (a0, a1) = self.split_mask32x8(a);
5943 let (b0, b1) = self.split_i32x8(b);
5944 let (c0, c1) = self.split_i32x8(c);
5945 self.combine_i32x4(self.select_i32x4(a0, b0, c0), self.select_i32x4(a1, b1, c1))
5946 }
5947 #[inline(always)]
5948 fn min_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5949 let (a0, a1) = self.split_i32x8(a);
5950 let (b0, b1) = self.split_i32x8(b);
5951 self.combine_i32x4(self.min_i32x4(a0, b0), self.min_i32x4(a1, b1))
5952 }
5953 #[inline(always)]
5954 fn max_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
5955 let (a0, a1) = self.split_i32x8(a);
5956 let (b0, b1) = self.split_i32x8(b);
5957 self.combine_i32x4(self.max_i32x4(a0, b0), self.max_i32x4(a1, b1))
5958 }
5959 #[inline(always)]
5960 fn combine_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x16<Self> {
5961 i32x16 {
5962 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
5963 simd: self,
5964 }
5965 }
5966 #[inline(always)]
5967 fn split_i32x8(self, a: i32x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
5968 (
5969 i32x4 {
5970 val: crate::support::Aligned128(a.val.0[0]),
5971 simd: self,
5972 },
5973 i32x4 {
5974 val: crate::support::Aligned128(a.val.0[1]),
5975 simd: self,
5976 },
5977 )
5978 }
5979 #[inline(always)]
5980 fn neg_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
5981 let (a0, a1) = self.split_i32x8(a);
5982 self.combine_i32x4(self.neg_i32x4(a0), self.neg_i32x4(a1))
5983 }
5984 #[inline(always)]
5985 fn reinterpret_u8_i32x8(self, a: i32x8<Self>) -> u8x32<Self> {
5986 let (a0, a1) = self.split_i32x8(a);
5987 self.combine_u8x16(self.reinterpret_u8_i32x4(a0), self.reinterpret_u8_i32x4(a1))
5988 }
5989 #[inline(always)]
5990 fn reinterpret_u32_i32x8(self, a: i32x8<Self>) -> u32x8<Self> {
5991 let (a0, a1) = self.split_i32x8(a);
5992 self.combine_u32x4(
5993 self.reinterpret_u32_i32x4(a0),
5994 self.reinterpret_u32_i32x4(a1),
5995 )
5996 }
5997 #[inline(always)]
5998 fn cvt_f32_i32x8(self, a: i32x8<Self>) -> f32x8<Self> {
5999 let (a0, a1) = self.split_i32x8(a);
6000 self.combine_f32x4(self.cvt_f32_i32x4(a0), self.cvt_f32_i32x4(a1))
6001 }
6002 #[inline(always)]
6003 fn splat_u32x8(self, val: u32) -> u32x8<Self> {
6004 let half = self.splat_u32x4(val);
6005 self.combine_u32x4(half, half)
6006 }
6007 #[inline(always)]
6008 fn load_array_u32x8(self, val: [u32; 8usize]) -> u32x8<Self> {
6009 u32x8 {
6010 val: crate::transmute::checked_transmute_copy(&val),
6011 simd: self,
6012 }
6013 }
6014 #[inline(always)]
6015 fn load_array_ref_u32x8(self, val: &[u32; 8usize]) -> u32x8<Self> {
6016 u32x8 {
6017 val: crate::transmute::checked_transmute_copy(val),
6018 simd: self,
6019 }
6020 }
6021 #[inline(always)]
6022 fn as_array_u32x8(self, a: u32x8<Self>) -> [u32; 8usize] {
6023 crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [u32; 8usize]>(&a.val.0)
6024 }
6025 #[inline(always)]
6026 fn as_array_ref_u32x8(self, a: &u32x8<Self>) -> &[u32; 8usize] {
6027 crate::transmute::checked_cast_ref::<[__m128i; 2usize], [u32; 8usize]>(&a.val.0)
6028 }
6029 #[inline(always)]
6030 fn as_array_mut_u32x8(self, a: &mut u32x8<Self>) -> &mut [u32; 8usize] {
6031 crate::transmute::checked_cast_mut::<[__m128i; 2usize], [u32; 8usize]>(&mut a.val.0)
6032 }
6033 #[inline(always)]
6034 fn store_array_u32x8(self, a: u32x8<Self>, dest: &mut [u32; 8usize]) -> () {
6035 crate::transmute::checked_transmute_store(a.val.0, dest);
6036 }
6037 #[inline(always)]
6038 fn cvt_from_bytes_u32x8(self, a: u8x32<Self>) -> u32x8<Self> {
6039 u32x8 {
6040 val: crate::transmute::checked_transmute_copy(&a.val),
6041 simd: self,
6042 }
6043 }
6044 #[inline(always)]
6045 fn cvt_to_bytes_u32x8(self, a: u32x8<Self>) -> u8x32<Self> {
6046 u8x32 {
6047 val: crate::transmute::checked_transmute_copy(&a.val),
6048 simd: self,
6049 }
6050 }
6051 #[inline(always)]
6052 fn slide_u32x8<const SHIFT: usize>(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6053 if SHIFT >= 8usize {
6054 return b;
6055 }
6056 let result = cross_block_alignr_128x2(
6057 self,
6058 self.cvt_to_bytes_u32x8(b).val.0,
6059 self.cvt_to_bytes_u32x8(a).val.0,
6060 SHIFT * 4usize,
6061 );
6062 self.cvt_from_bytes_u32x8(u8x32 {
6063 val: crate::support::Aligned256(result),
6064 simd: self,
6065 })
6066 }
6067 #[inline(always)]
6068 fn slide_within_blocks_u32x8<const SHIFT: usize>(
6069 self,
6070 a: u32x8<Self>,
6071 b: u32x8<Self>,
6072 ) -> u32x8<Self> {
6073 let (a0, a1) = self.split_u32x8(a);
6074 let (b0, b1) = self.split_u32x8(b);
6075 self.combine_u32x4(
6076 self.slide_within_blocks_u32x4::<SHIFT>(a0, b0),
6077 self.slide_within_blocks_u32x4::<SHIFT>(a1, b1),
6078 )
6079 }
6080 #[inline(always)]
6081 fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8<Self>, indices: u8x32<Self>) -> u32x8<Self> {
6082 let (a0, a1) = self.split_u32x8(a);
6083 let (indices0, indices1) = self.split_u8x32(indices);
6084 self.combine_u32x4(
6085 self.swizzle_dyn_within_blocks_u32x4(a0, indices0),
6086 self.swizzle_dyn_within_blocks_u32x4(a1, indices1),
6087 )
6088 }
6089 #[inline(always)]
6090 fn add_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6091 let (a0, a1) = self.split_u32x8(a);
6092 let (b0, b1) = self.split_u32x8(b);
6093 self.combine_u32x4(self.add_u32x4(a0, b0), self.add_u32x4(a1, b1))
6094 }
6095 #[inline(always)]
6096 fn sub_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6097 let (a0, a1) = self.split_u32x8(a);
6098 let (b0, b1) = self.split_u32x8(b);
6099 self.combine_u32x4(self.sub_u32x4(a0, b0), self.sub_u32x4(a1, b1))
6100 }
6101 #[inline(always)]
6102 fn mul_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6103 let (a0, a1) = self.split_u32x8(a);
6104 let (b0, b1) = self.split_u32x8(b);
6105 self.combine_u32x4(self.mul_u32x4(a0, b0), self.mul_u32x4(a1, b1))
6106 }
6107 #[inline(always)]
6108 fn and_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6109 let (a0, a1) = self.split_u32x8(a);
6110 let (b0, b1) = self.split_u32x8(b);
6111 self.combine_u32x4(self.and_u32x4(a0, b0), self.and_u32x4(a1, b1))
6112 }
6113 #[inline(always)]
6114 fn or_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6115 let (a0, a1) = self.split_u32x8(a);
6116 let (b0, b1) = self.split_u32x8(b);
6117 self.combine_u32x4(self.or_u32x4(a0, b0), self.or_u32x4(a1, b1))
6118 }
6119 #[inline(always)]
6120 fn xor_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6121 let (a0, a1) = self.split_u32x8(a);
6122 let (b0, b1) = self.split_u32x8(b);
6123 self.combine_u32x4(self.xor_u32x4(a0, b0), self.xor_u32x4(a1, b1))
6124 }
6125 #[inline(always)]
6126 fn not_u32x8(self, a: u32x8<Self>) -> u32x8<Self> {
6127 let (a0, a1) = self.split_u32x8(a);
6128 self.combine_u32x4(self.not_u32x4(a0), self.not_u32x4(a1))
6129 }
6130 #[inline(always)]
6131 fn shl_u32x8(self, a: u32x8<Self>, shift: u32) -> u32x8<Self> {
6132 let (a0, a1) = self.split_u32x8(a);
6133 self.combine_u32x4(self.shl_u32x4(a0, shift), self.shl_u32x4(a1, shift))
6134 }
6135 #[inline(always)]
6136 fn shlv_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6137 let (a0, a1) = self.split_u32x8(a);
6138 let (b0, b1) = self.split_u32x8(b);
6139 self.combine_u32x4(self.shlv_u32x4(a0, b0), self.shlv_u32x4(a1, b1))
6140 }
6141 #[inline(always)]
6142 fn shr_u32x8(self, a: u32x8<Self>, shift: u32) -> u32x8<Self> {
6143 let (a0, a1) = self.split_u32x8(a);
6144 self.combine_u32x4(self.shr_u32x4(a0, shift), self.shr_u32x4(a1, shift))
6145 }
6146 #[inline(always)]
6147 fn shrv_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6148 let (a0, a1) = self.split_u32x8(a);
6149 let (b0, b1) = self.split_u32x8(b);
6150 self.combine_u32x4(self.shrv_u32x4(a0, b0), self.shrv_u32x4(a1, b1))
6151 }
6152 #[inline(always)]
6153 fn simd_eq_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
6154 let (a0, a1) = self.split_u32x8(a);
6155 let (b0, b1) = self.split_u32x8(b);
6156 self.combine_mask32x4(self.simd_eq_u32x4(a0, b0), self.simd_eq_u32x4(a1, b1))
6157 }
6158 #[inline(always)]
6159 fn simd_lt_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
6160 let (a0, a1) = self.split_u32x8(a);
6161 let (b0, b1) = self.split_u32x8(b);
6162 self.combine_mask32x4(self.simd_lt_u32x4(a0, b0), self.simd_lt_u32x4(a1, b1))
6163 }
6164 #[inline(always)]
6165 fn simd_le_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
6166 let (a0, a1) = self.split_u32x8(a);
6167 let (b0, b1) = self.split_u32x8(b);
6168 self.combine_mask32x4(self.simd_le_u32x4(a0, b0), self.simd_le_u32x4(a1, b1))
6169 }
6170 #[inline(always)]
6171 fn simd_ge_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
6172 let (a0, a1) = self.split_u32x8(a);
6173 let (b0, b1) = self.split_u32x8(b);
6174 self.combine_mask32x4(self.simd_ge_u32x4(a0, b0), self.simd_ge_u32x4(a1, b1))
6175 }
6176 #[inline(always)]
6177 fn simd_gt_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
6178 let (a0, a1) = self.split_u32x8(a);
6179 let (b0, b1) = self.split_u32x8(b);
6180 self.combine_mask32x4(self.simd_gt_u32x4(a0, b0), self.simd_gt_u32x4(a1, b1))
6181 }
6182 #[inline(always)]
6183 fn zip_low_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6184 let (a0, _) = self.split_u32x8(a);
6185 let (b0, _) = self.split_u32x8(b);
6186 self.combine_u32x4(self.zip_low_u32x4(a0, b0), self.zip_high_u32x4(a0, b0))
6187 }
6188 #[inline(always)]
6189 fn zip_high_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6190 let (_, a1) = self.split_u32x8(a);
6191 let (_, b1) = self.split_u32x8(b);
6192 self.combine_u32x4(self.zip_low_u32x4(a1, b1), self.zip_high_u32x4(a1, b1))
6193 }
6194 #[inline(always)]
6195 fn unzip_low_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6196 let (a0, a1) = self.split_u32x8(a);
6197 let (b0, b1) = self.split_u32x8(b);
6198 self.combine_u32x4(self.unzip_low_u32x4(a0, a1), self.unzip_low_u32x4(b0, b1))
6199 }
6200 #[inline(always)]
6201 fn unzip_high_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6202 let (a0, a1) = self.split_u32x8(a);
6203 let (b0, b1) = self.split_u32x8(b);
6204 self.combine_u32x4(self.unzip_high_u32x4(a0, a1), self.unzip_high_u32x4(b0, b1))
6205 }
6206 #[inline(always)]
6207 fn interleave_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u32x8<Self>, u32x8<Self>) {
6208 let (a0, a1) = self.split_u32x8(a);
6209 let (b0, b1) = self.split_u32x8(b);
6210 let lo_lo = self.zip_low_u32x4(a0, b0);
6211 let lo_hi = self.zip_high_u32x4(a0, b0);
6212 let hi_lo = self.zip_low_u32x4(a1, b1);
6213 let hi_hi = self.zip_high_u32x4(a1, b1);
6214 (
6215 self.combine_u32x4(lo_lo, lo_hi),
6216 self.combine_u32x4(hi_lo, hi_hi),
6217 )
6218 }
6219 #[inline(always)]
6220 fn deinterleave_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u32x8<Self>, u32x8<Self>) {
6221 let (a0, a1) = self.split_u32x8(a);
6222 let (b0, b1) = self.split_u32x8(b);
6223 let lo_even = self.unzip_low_u32x4(a0, a1);
6224 let lo_odd = self.unzip_high_u32x4(a0, a1);
6225 let hi_even = self.unzip_low_u32x4(b0, b1);
6226 let hi_odd = self.unzip_high_u32x4(b0, b1);
6227 (
6228 self.combine_u32x4(lo_even, hi_even),
6229 self.combine_u32x4(lo_odd, hi_odd),
6230 )
6231 }
6232 #[inline(always)]
6233 fn select_u32x8(self, a: mask32x8<Self>, b: u32x8<Self>, c: u32x8<Self>) -> u32x8<Self> {
6234 let (a0, a1) = self.split_mask32x8(a);
6235 let (b0, b1) = self.split_u32x8(b);
6236 let (c0, c1) = self.split_u32x8(c);
6237 self.combine_u32x4(self.select_u32x4(a0, b0, c0), self.select_u32x4(a1, b1, c1))
6238 }
6239 #[inline(always)]
6240 fn min_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6241 let (a0, a1) = self.split_u32x8(a);
6242 let (b0, b1) = self.split_u32x8(b);
6243 self.combine_u32x4(self.min_u32x4(a0, b0), self.min_u32x4(a1, b1))
6244 }
6245 #[inline(always)]
6246 fn max_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
6247 let (a0, a1) = self.split_u32x8(a);
6248 let (b0, b1) = self.split_u32x8(b);
6249 self.combine_u32x4(self.max_u32x4(a0, b0), self.max_u32x4(a1, b1))
6250 }
6251 #[inline(always)]
6252 fn combine_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x16<Self> {
6253 u32x16 {
6254 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
6255 simd: self,
6256 }
6257 }
6258 #[inline(always)]
6259 fn split_u32x8(self, a: u32x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
6260 (
6261 u32x4 {
6262 val: crate::support::Aligned128(a.val.0[0]),
6263 simd: self,
6264 },
6265 u32x4 {
6266 val: crate::support::Aligned128(a.val.0[1]),
6267 simd: self,
6268 },
6269 )
6270 }
6271 #[inline(always)]
6272 fn reinterpret_u8_u32x8(self, a: u32x8<Self>) -> u8x32<Self> {
6273 let (a0, a1) = self.split_u32x8(a);
6274 self.combine_u8x16(self.reinterpret_u8_u32x4(a0), self.reinterpret_u8_u32x4(a1))
6275 }
6276 #[inline(always)]
6277 fn cvt_f32_u32x8(self, a: u32x8<Self>) -> f32x8<Self> {
6278 let (a0, a1) = self.split_u32x8(a);
6279 self.combine_f32x4(self.cvt_f32_u32x4(a0), self.cvt_f32_u32x4(a1))
6280 }
6281 #[inline(always)]
6282 fn splat_mask32x8(self, val: bool) -> mask32x8<Self> {
6283 let half = self.splat_mask32x4(val);
6284 self.combine_mask32x4(half, half)
6285 }
6286 #[inline(always)]
6287 fn load_array_mask32x8(self, val: [i32; 8usize]) -> mask32x8<Self> {
6288 mask32x8 {
6289 val: crate::transmute::checked_transmute_copy(&val),
6290 simd: self,
6291 }
6292 }
6293 #[inline(always)]
6294 fn as_array_mask32x8(self, a: mask32x8<Self>) -> [i32; 8usize] {
6295 crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i32; 8usize]>(&a.val.0)
6296 }
6297 #[inline(always)]
6298 fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8<Self> {
6299 let lo = self.from_bitmask_mask32x4(bits);
6300 let hi = self.from_bitmask_mask32x4(bits >> 4usize);
6301 self.combine_mask32x4(lo, hi)
6302 }
6303 #[inline(always)]
6304 fn to_bitmask_mask32x8(self, a: mask32x8<Self>) -> u64 {
6305 let (lo, hi) = self.split_mask32x8(a);
6306 let lo = self.to_bitmask_mask32x4(lo);
6307 let hi = self.to_bitmask_mask32x4(hi);
6308 lo | (hi << 4usize)
6309 }
6310 #[inline(always)]
6311 fn set_mask32x8(self, a: &mut mask32x8<Self>, index: usize, value: bool) -> () {
6312 assert!(
6313 index < 8usize,
6314 "mask lane index {index} is out of bounds for {} lanes",
6315 8usize
6316 );
6317 let mut lanes = self.as_array_mask32x8(*a);
6318 lanes[index] = if value { !0 } else { 0 };
6319 *a = self.load_array_mask32x8(lanes);
6320 }
6321 #[inline(always)]
6322 fn and_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
6323 let (a0, a1) = self.split_mask32x8(a);
6324 let (b0, b1) = self.split_mask32x8(b);
6325 self.combine_mask32x4(self.and_mask32x4(a0, b0), self.and_mask32x4(a1, b1))
6326 }
6327 #[inline(always)]
6328 fn or_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
6329 let (a0, a1) = self.split_mask32x8(a);
6330 let (b0, b1) = self.split_mask32x8(b);
6331 self.combine_mask32x4(self.or_mask32x4(a0, b0), self.or_mask32x4(a1, b1))
6332 }
6333 #[inline(always)]
6334 fn xor_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
6335 let (a0, a1) = self.split_mask32x8(a);
6336 let (b0, b1) = self.split_mask32x8(b);
6337 self.combine_mask32x4(self.xor_mask32x4(a0, b0), self.xor_mask32x4(a1, b1))
6338 }
6339 #[inline(always)]
6340 fn not_mask32x8(self, a: mask32x8<Self>) -> mask32x8<Self> {
6341 let (a0, a1) = self.split_mask32x8(a);
6342 self.combine_mask32x4(self.not_mask32x4(a0), self.not_mask32x4(a1))
6343 }
6344 #[inline(always)]
6345 fn select_mask32x8(
6346 self,
6347 a: mask32x8<Self>,
6348 b: mask32x8<Self>,
6349 c: mask32x8<Self>,
6350 ) -> mask32x8<Self> {
6351 let (a0, a1) = self.split_mask32x8(a);
6352 let (b0, b1) = self.split_mask32x8(b);
6353 let (c0, c1) = self.split_mask32x8(c);
6354 self.combine_mask32x4(
6355 self.select_mask32x4(a0, b0, c0),
6356 self.select_mask32x4(a1, b1, c1),
6357 )
6358 }
6359 #[inline(always)]
6360 fn simd_eq_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
6361 let (a0, a1) = self.split_mask32x8(a);
6362 let (b0, b1) = self.split_mask32x8(b);
6363 self.combine_mask32x4(self.simd_eq_mask32x4(a0, b0), self.simd_eq_mask32x4(a1, b1))
6364 }
6365 #[inline(always)]
6366 fn any_true_mask32x8(self, a: mask32x8<Self>) -> bool {
6367 let (a0, a1) = self.split_mask32x8(a);
6368 self.any_true_mask32x4(a0) || self.any_true_mask32x4(a1)
6369 }
6370 #[inline(always)]
6371 fn all_true_mask32x8(self, a: mask32x8<Self>) -> bool {
6372 let (a0, a1) = self.split_mask32x8(a);
6373 self.all_true_mask32x4(a0) && self.all_true_mask32x4(a1)
6374 }
6375 #[inline(always)]
6376 fn any_false_mask32x8(self, a: mask32x8<Self>) -> bool {
6377 let (a0, a1) = self.split_mask32x8(a);
6378 self.any_false_mask32x4(a0) || self.any_false_mask32x4(a1)
6379 }
6380 #[inline(always)]
6381 fn all_false_mask32x8(self, a: mask32x8<Self>) -> bool {
6382 let (a0, a1) = self.split_mask32x8(a);
6383 self.all_false_mask32x4(a0) && self.all_false_mask32x4(a1)
6384 }
6385 #[inline(always)]
6386 fn combine_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x16<Self> {
6387 mask32x16 {
6388 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
6389 simd: self,
6390 }
6391 }
6392 #[inline(always)]
6393 fn split_mask32x8(self, a: mask32x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
6394 (
6395 mask32x4 {
6396 val: crate::support::Aligned128(a.val.0[0]),
6397 simd: self,
6398 },
6399 mask32x4 {
6400 val: crate::support::Aligned128(a.val.0[1]),
6401 simd: self,
6402 },
6403 )
6404 }
6405 #[inline(always)]
6406 fn splat_f64x4(self, val: f64) -> f64x4<Self> {
6407 let half = self.splat_f64x2(val);
6408 self.combine_f64x2(half, half)
6409 }
6410 #[inline(always)]
6411 fn load_array_f64x4(self, val: [f64; 4usize]) -> f64x4<Self> {
6412 f64x4 {
6413 val: crate::transmute::checked_transmute_copy(&val),
6414 simd: self,
6415 }
6416 }
6417 #[inline(always)]
6418 fn load_array_ref_f64x4(self, val: &[f64; 4usize]) -> f64x4<Self> {
6419 f64x4 {
6420 val: crate::transmute::checked_transmute_copy(val),
6421 simd: self,
6422 }
6423 }
6424 #[inline(always)]
6425 fn as_array_f64x4(self, a: f64x4<Self>) -> [f64; 4usize] {
6426 crate::transmute::checked_transmute_copy::<[__m128d; 2usize], [f64; 4usize]>(&a.val.0)
6427 }
6428 #[inline(always)]
6429 fn as_array_ref_f64x4(self, a: &f64x4<Self>) -> &[f64; 4usize] {
6430 crate::transmute::checked_cast_ref::<[__m128d; 2usize], [f64; 4usize]>(&a.val.0)
6431 }
6432 #[inline(always)]
6433 fn as_array_mut_f64x4(self, a: &mut f64x4<Self>) -> &mut [f64; 4usize] {
6434 crate::transmute::checked_cast_mut::<[__m128d; 2usize], [f64; 4usize]>(&mut a.val.0)
6435 }
6436 #[inline(always)]
6437 fn store_array_f64x4(self, a: f64x4<Self>, dest: &mut [f64; 4usize]) -> () {
6438 crate::transmute::checked_transmute_store(a.val.0, dest);
6439 }
6440 #[inline(always)]
6441 fn cvt_from_bytes_f64x4(self, a: u8x32<Self>) -> f64x4<Self> {
6442 f64x4 {
6443 val: crate::transmute::checked_transmute_copy(&a.val),
6444 simd: self,
6445 }
6446 }
6447 #[inline(always)]
6448 fn cvt_to_bytes_f64x4(self, a: f64x4<Self>) -> u8x32<Self> {
6449 u8x32 {
6450 val: crate::transmute::checked_transmute_copy(&a.val),
6451 simd: self,
6452 }
6453 }
6454 #[inline(always)]
6455 fn slide_f64x4<const SHIFT: usize>(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6456 if SHIFT >= 4usize {
6457 return b;
6458 }
6459 let result = cross_block_alignr_128x2(
6460 self,
6461 self.cvt_to_bytes_f64x4(b).val.0,
6462 self.cvt_to_bytes_f64x4(a).val.0,
6463 SHIFT * 8usize,
6464 );
6465 self.cvt_from_bytes_f64x4(u8x32 {
6466 val: crate::support::Aligned256(result),
6467 simd: self,
6468 })
6469 }
6470 #[inline(always)]
6471 fn slide_within_blocks_f64x4<const SHIFT: usize>(
6472 self,
6473 a: f64x4<Self>,
6474 b: f64x4<Self>,
6475 ) -> f64x4<Self> {
6476 let (a0, a1) = self.split_f64x4(a);
6477 let (b0, b1) = self.split_f64x4(b);
6478 self.combine_f64x2(
6479 self.slide_within_blocks_f64x2::<SHIFT>(a0, b0),
6480 self.slide_within_blocks_f64x2::<SHIFT>(a1, b1),
6481 )
6482 }
6483 #[inline(always)]
6484 fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4<Self>, indices: u8x32<Self>) -> f64x4<Self> {
6485 let (a0, a1) = self.split_f64x4(a);
6486 let (indices0, indices1) = self.split_u8x32(indices);
6487 self.combine_f64x2(
6488 self.swizzle_dyn_within_blocks_f64x2(a0, indices0),
6489 self.swizzle_dyn_within_blocks_f64x2(a1, indices1),
6490 )
6491 }
6492 #[inline(always)]
6493 fn abs_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
6494 let (a0, a1) = self.split_f64x4(a);
6495 self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1))
6496 }
6497 #[inline(always)]
6498 fn neg_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
6499 let (a0, a1) = self.split_f64x4(a);
6500 self.combine_f64x2(self.neg_f64x2(a0), self.neg_f64x2(a1))
6501 }
6502 #[inline(always)]
6503 fn sqrt_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
6504 let (a0, a1) = self.split_f64x4(a);
6505 self.combine_f64x2(self.sqrt_f64x2(a0), self.sqrt_f64x2(a1))
6506 }
6507 #[inline(always)]
6508 fn approximate_recip_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
6509 let (a0, a1) = self.split_f64x4(a);
6510 self.combine_f64x2(
6511 self.approximate_recip_f64x2(a0),
6512 self.approximate_recip_f64x2(a1),
6513 )
6514 }
6515 #[inline(always)]
6516 fn add_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6517 let (a0, a1) = self.split_f64x4(a);
6518 let (b0, b1) = self.split_f64x4(b);
6519 self.combine_f64x2(self.add_f64x2(a0, b0), self.add_f64x2(a1, b1))
6520 }
6521 #[inline(always)]
6522 fn sub_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6523 let (a0, a1) = self.split_f64x4(a);
6524 let (b0, b1) = self.split_f64x4(b);
6525 self.combine_f64x2(self.sub_f64x2(a0, b0), self.sub_f64x2(a1, b1))
6526 }
6527 #[inline(always)]
6528 fn mul_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6529 let (a0, a1) = self.split_f64x4(a);
6530 let (b0, b1) = self.split_f64x4(b);
6531 self.combine_f64x2(self.mul_f64x2(a0, b0), self.mul_f64x2(a1, b1))
6532 }
6533 #[inline(always)]
6534 fn div_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6535 let (a0, a1) = self.split_f64x4(a);
6536 let (b0, b1) = self.split_f64x4(b);
6537 self.combine_f64x2(self.div_f64x2(a0, b0), self.div_f64x2(a1, b1))
6538 }
6539 #[inline(always)]
6540 fn copysign_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6541 let (a0, a1) = self.split_f64x4(a);
6542 let (b0, b1) = self.split_f64x4(b);
6543 self.combine_f64x2(self.copysign_f64x2(a0, b0), self.copysign_f64x2(a1, b1))
6544 }
6545 #[inline(always)]
6546 fn simd_eq_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
6547 let (a0, a1) = self.split_f64x4(a);
6548 let (b0, b1) = self.split_f64x4(b);
6549 self.combine_mask64x2(self.simd_eq_f64x2(a0, b0), self.simd_eq_f64x2(a1, b1))
6550 }
6551 #[inline(always)]
6552 fn simd_lt_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
6553 let (a0, a1) = self.split_f64x4(a);
6554 let (b0, b1) = self.split_f64x4(b);
6555 self.combine_mask64x2(self.simd_lt_f64x2(a0, b0), self.simd_lt_f64x2(a1, b1))
6556 }
6557 #[inline(always)]
6558 fn simd_le_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
6559 let (a0, a1) = self.split_f64x4(a);
6560 let (b0, b1) = self.split_f64x4(b);
6561 self.combine_mask64x2(self.simd_le_f64x2(a0, b0), self.simd_le_f64x2(a1, b1))
6562 }
6563 #[inline(always)]
6564 fn simd_ge_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
6565 let (a0, a1) = self.split_f64x4(a);
6566 let (b0, b1) = self.split_f64x4(b);
6567 self.combine_mask64x2(self.simd_ge_f64x2(a0, b0), self.simd_ge_f64x2(a1, b1))
6568 }
6569 #[inline(always)]
6570 fn simd_gt_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
6571 let (a0, a1) = self.split_f64x4(a);
6572 let (b0, b1) = self.split_f64x4(b);
6573 self.combine_mask64x2(self.simd_gt_f64x2(a0, b0), self.simd_gt_f64x2(a1, b1))
6574 }
6575 #[inline(always)]
6576 fn zip_low_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6577 let (a0, _) = self.split_f64x4(a);
6578 let (b0, _) = self.split_f64x4(b);
6579 self.combine_f64x2(self.zip_low_f64x2(a0, b0), self.zip_high_f64x2(a0, b0))
6580 }
6581 #[inline(always)]
6582 fn zip_high_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6583 let (_, a1) = self.split_f64x4(a);
6584 let (_, b1) = self.split_f64x4(b);
6585 self.combine_f64x2(self.zip_low_f64x2(a1, b1), self.zip_high_f64x2(a1, b1))
6586 }
6587 #[inline(always)]
6588 fn unzip_low_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6589 let (a0, a1) = self.split_f64x4(a);
6590 let (b0, b1) = self.split_f64x4(b);
6591 self.combine_f64x2(self.unzip_low_f64x2(a0, a1), self.unzip_low_f64x2(b0, b1))
6592 }
6593 #[inline(always)]
6594 fn unzip_high_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6595 let (a0, a1) = self.split_f64x4(a);
6596 let (b0, b1) = self.split_f64x4(b);
6597 self.combine_f64x2(self.unzip_high_f64x2(a0, a1), self.unzip_high_f64x2(b0, b1))
6598 }
6599 #[inline(always)]
6600 fn interleave_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> (f64x4<Self>, f64x4<Self>) {
6601 let (a0, a1) = self.split_f64x4(a);
6602 let (b0, b1) = self.split_f64x4(b);
6603 let lo_lo = self.zip_low_f64x2(a0, b0);
6604 let lo_hi = self.zip_high_f64x2(a0, b0);
6605 let hi_lo = self.zip_low_f64x2(a1, b1);
6606 let hi_hi = self.zip_high_f64x2(a1, b1);
6607 (
6608 self.combine_f64x2(lo_lo, lo_hi),
6609 self.combine_f64x2(hi_lo, hi_hi),
6610 )
6611 }
6612 #[inline(always)]
6613 fn deinterleave_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> (f64x4<Self>, f64x4<Self>) {
6614 let (a0, a1) = self.split_f64x4(a);
6615 let (b0, b1) = self.split_f64x4(b);
6616 let lo_even = self.unzip_low_f64x2(a0, a1);
6617 let lo_odd = self.unzip_high_f64x2(a0, a1);
6618 let hi_even = self.unzip_low_f64x2(b0, b1);
6619 let hi_odd = self.unzip_high_f64x2(b0, b1);
6620 (
6621 self.combine_f64x2(lo_even, hi_even),
6622 self.combine_f64x2(lo_odd, hi_odd),
6623 )
6624 }
6625 #[inline(always)]
6626 fn max_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6627 let (a0, a1) = self.split_f64x4(a);
6628 let (b0, b1) = self.split_f64x4(b);
6629 self.combine_f64x2(self.max_f64x2(a0, b0), self.max_f64x2(a1, b1))
6630 }
6631 #[inline(always)]
6632 fn min_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6633 let (a0, a1) = self.split_f64x4(a);
6634 let (b0, b1) = self.split_f64x4(b);
6635 self.combine_f64x2(self.min_f64x2(a0, b0), self.min_f64x2(a1, b1))
6636 }
6637 #[inline(always)]
6638 fn max_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6639 let (a0, a1) = self.split_f64x4(a);
6640 let (b0, b1) = self.split_f64x4(b);
6641 self.combine_f64x2(
6642 self.max_precise_f64x2(a0, b0),
6643 self.max_precise_f64x2(a1, b1),
6644 )
6645 }
6646 #[inline(always)]
6647 fn min_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
6648 let (a0, a1) = self.split_f64x4(a);
6649 let (b0, b1) = self.split_f64x4(b);
6650 self.combine_f64x2(
6651 self.min_precise_f64x2(a0, b0),
6652 self.min_precise_f64x2(a1, b1),
6653 )
6654 }
6655 #[inline(always)]
6656 fn mul_add_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
6657 let (a0, a1) = self.split_f64x4(a);
6658 let (b0, b1) = self.split_f64x4(b);
6659 let (c0, c1) = self.split_f64x4(c);
6660 self.combine_f64x2(
6661 self.mul_add_f64x2(a0, b0, c0),
6662 self.mul_add_f64x2(a1, b1, c1),
6663 )
6664 }
6665 #[inline(always)]
6666 fn mul_sub_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
6667 let (a0, a1) = self.split_f64x4(a);
6668 let (b0, b1) = self.split_f64x4(b);
6669 let (c0, c1) = self.split_f64x4(c);
6670 self.combine_f64x2(
6671 self.mul_sub_f64x2(a0, b0, c0),
6672 self.mul_sub_f64x2(a1, b1, c1),
6673 )
6674 }
6675 #[inline(always)]
6676 fn floor_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
6677 let (a0, a1) = self.split_f64x4(a);
6678 self.combine_f64x2(self.floor_f64x2(a0), self.floor_f64x2(a1))
6679 }
6680 #[inline(always)]
6681 fn ceil_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
6682 let (a0, a1) = self.split_f64x4(a);
6683 self.combine_f64x2(self.ceil_f64x2(a0), self.ceil_f64x2(a1))
6684 }
6685 #[inline(always)]
6686 fn round_ties_even_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
6687 let (a0, a1) = self.split_f64x4(a);
6688 self.combine_f64x2(
6689 self.round_ties_even_f64x2(a0),
6690 self.round_ties_even_f64x2(a1),
6691 )
6692 }
6693 #[inline(always)]
6694 fn fract_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
6695 let (a0, a1) = self.split_f64x4(a);
6696 self.combine_f64x2(self.fract_f64x2(a0), self.fract_f64x2(a1))
6697 }
6698 #[inline(always)]
6699 fn trunc_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
6700 let (a0, a1) = self.split_f64x4(a);
6701 self.combine_f64x2(self.trunc_f64x2(a0), self.trunc_f64x2(a1))
6702 }
6703 #[inline(always)]
6704 fn select_f64x4(self, a: mask64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
6705 let (a0, a1) = self.split_mask64x4(a);
6706 let (b0, b1) = self.split_f64x4(b);
6707 let (c0, c1) = self.split_f64x4(c);
6708 self.combine_f64x2(self.select_f64x2(a0, b0, c0), self.select_f64x2(a1, b1, c1))
6709 }
6710 #[inline(always)]
6711 fn combine_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x8<Self> {
6712 f64x8 {
6713 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
6714 simd: self,
6715 }
6716 }
6717 #[inline(always)]
6718 fn split_f64x4(self, a: f64x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
6719 (
6720 f64x2 {
6721 val: crate::support::Aligned128(a.val.0[0]),
6722 simd: self,
6723 },
6724 f64x2 {
6725 val: crate::support::Aligned128(a.val.0[1]),
6726 simd: self,
6727 },
6728 )
6729 }
6730 #[inline(always)]
6731 fn reinterpret_f32_f64x4(self, a: f64x4<Self>) -> f32x8<Self> {
6732 let (a0, a1) = self.split_f64x4(a);
6733 self.combine_f32x4(
6734 self.reinterpret_f32_f64x2(a0),
6735 self.reinterpret_f32_f64x2(a1),
6736 )
6737 }
6738 #[inline(always)]
6739 fn splat_mask64x4(self, val: bool) -> mask64x4<Self> {
6740 let half = self.splat_mask64x2(val);
6741 self.combine_mask64x2(half, half)
6742 }
6743 #[inline(always)]
6744 fn load_array_mask64x4(self, val: [i64; 4usize]) -> mask64x4<Self> {
6745 mask64x4 {
6746 val: crate::transmute::checked_transmute_copy(&val),
6747 simd: self,
6748 }
6749 }
6750 #[inline(always)]
6751 fn as_array_mask64x4(self, a: mask64x4<Self>) -> [i64; 4usize] {
6752 crate::transmute::checked_transmute_copy::<[__m128i; 2usize], [i64; 4usize]>(&a.val.0)
6753 }
6754 #[inline(always)]
6755 fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4<Self> {
6756 let lo = self.from_bitmask_mask64x2(bits);
6757 let hi = self.from_bitmask_mask64x2(bits >> 2usize);
6758 self.combine_mask64x2(lo, hi)
6759 }
6760 #[inline(always)]
6761 fn to_bitmask_mask64x4(self, a: mask64x4<Self>) -> u64 {
6762 let (lo, hi) = self.split_mask64x4(a);
6763 let lo = self.to_bitmask_mask64x2(lo);
6764 let hi = self.to_bitmask_mask64x2(hi);
6765 lo | (hi << 2usize)
6766 }
6767 #[inline(always)]
6768 fn set_mask64x4(self, a: &mut mask64x4<Self>, index: usize, value: bool) -> () {
6769 assert!(
6770 index < 4usize,
6771 "mask lane index {index} is out of bounds for {} lanes",
6772 4usize
6773 );
6774 let mut lanes = self.as_array_mask64x4(*a);
6775 lanes[index] = if value { !0 } else { 0 };
6776 *a = self.load_array_mask64x4(lanes);
6777 }
6778 #[inline(always)]
6779 fn and_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
6780 let (a0, a1) = self.split_mask64x4(a);
6781 let (b0, b1) = self.split_mask64x4(b);
6782 self.combine_mask64x2(self.and_mask64x2(a0, b0), self.and_mask64x2(a1, b1))
6783 }
6784 #[inline(always)]
6785 fn or_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
6786 let (a0, a1) = self.split_mask64x4(a);
6787 let (b0, b1) = self.split_mask64x4(b);
6788 self.combine_mask64x2(self.or_mask64x2(a0, b0), self.or_mask64x2(a1, b1))
6789 }
6790 #[inline(always)]
6791 fn xor_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
6792 let (a0, a1) = self.split_mask64x4(a);
6793 let (b0, b1) = self.split_mask64x4(b);
6794 self.combine_mask64x2(self.xor_mask64x2(a0, b0), self.xor_mask64x2(a1, b1))
6795 }
6796 #[inline(always)]
6797 fn not_mask64x4(self, a: mask64x4<Self>) -> mask64x4<Self> {
6798 let (a0, a1) = self.split_mask64x4(a);
6799 self.combine_mask64x2(self.not_mask64x2(a0), self.not_mask64x2(a1))
6800 }
6801 #[inline(always)]
6802 fn select_mask64x4(
6803 self,
6804 a: mask64x4<Self>,
6805 b: mask64x4<Self>,
6806 c: mask64x4<Self>,
6807 ) -> mask64x4<Self> {
6808 let (a0, a1) = self.split_mask64x4(a);
6809 let (b0, b1) = self.split_mask64x4(b);
6810 let (c0, c1) = self.split_mask64x4(c);
6811 self.combine_mask64x2(
6812 self.select_mask64x2(a0, b0, c0),
6813 self.select_mask64x2(a1, b1, c1),
6814 )
6815 }
6816 #[inline(always)]
6817 fn simd_eq_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
6818 let (a0, a1) = self.split_mask64x4(a);
6819 let (b0, b1) = self.split_mask64x4(b);
6820 self.combine_mask64x2(self.simd_eq_mask64x2(a0, b0), self.simd_eq_mask64x2(a1, b1))
6821 }
6822 #[inline(always)]
6823 fn any_true_mask64x4(self, a: mask64x4<Self>) -> bool {
6824 let (a0, a1) = self.split_mask64x4(a);
6825 self.any_true_mask64x2(a0) || self.any_true_mask64x2(a1)
6826 }
6827 #[inline(always)]
6828 fn all_true_mask64x4(self, a: mask64x4<Self>) -> bool {
6829 let (a0, a1) = self.split_mask64x4(a);
6830 self.all_true_mask64x2(a0) && self.all_true_mask64x2(a1)
6831 }
6832 #[inline(always)]
6833 fn any_false_mask64x4(self, a: mask64x4<Self>) -> bool {
6834 let (a0, a1) = self.split_mask64x4(a);
6835 self.any_false_mask64x2(a0) || self.any_false_mask64x2(a1)
6836 }
6837 #[inline(always)]
6838 fn all_false_mask64x4(self, a: mask64x4<Self>) -> bool {
6839 let (a0, a1) = self.split_mask64x4(a);
6840 self.all_false_mask64x2(a0) && self.all_false_mask64x2(a1)
6841 }
6842 #[inline(always)]
6843 fn combine_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x8<Self> {
6844 mask64x8 {
6845 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
6846 simd: self,
6847 }
6848 }
6849 #[inline(always)]
6850 fn split_mask64x4(self, a: mask64x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
6851 (
6852 mask64x2 {
6853 val: crate::support::Aligned128(a.val.0[0]),
6854 simd: self,
6855 },
6856 mask64x2 {
6857 val: crate::support::Aligned128(a.val.0[1]),
6858 simd: self,
6859 },
6860 )
6861 }
6862 #[inline(always)]
6863 fn splat_f32x16(self, val: f32) -> f32x16<Self> {
6864 let half = self.splat_f32x8(val);
6865 self.combine_f32x8(half, half)
6866 }
6867 #[inline(always)]
6868 fn load_array_f32x16(self, val: [f32; 16usize]) -> f32x16<Self> {
6869 f32x16 {
6870 val: crate::transmute::checked_transmute_copy(&val),
6871 simd: self,
6872 }
6873 }
6874 #[inline(always)]
6875 fn load_array_ref_f32x16(self, val: &[f32; 16usize]) -> f32x16<Self> {
6876 f32x16 {
6877 val: crate::transmute::checked_transmute_copy(val),
6878 simd: self,
6879 }
6880 }
6881 #[inline(always)]
6882 fn as_array_f32x16(self, a: f32x16<Self>) -> [f32; 16usize] {
6883 crate::transmute::checked_transmute_copy::<[__m128; 4usize], [f32; 16usize]>(&a.val.0)
6884 }
6885 #[inline(always)]
6886 fn as_array_ref_f32x16(self, a: &f32x16<Self>) -> &[f32; 16usize] {
6887 crate::transmute::checked_cast_ref::<[__m128; 4usize], [f32; 16usize]>(&a.val.0)
6888 }
6889 #[inline(always)]
6890 fn as_array_mut_f32x16(self, a: &mut f32x16<Self>) -> &mut [f32; 16usize] {
6891 crate::transmute::checked_cast_mut::<[__m128; 4usize], [f32; 16usize]>(&mut a.val.0)
6892 }
6893 #[inline(always)]
6894 fn store_array_f32x16(self, a: f32x16<Self>, dest: &mut [f32; 16usize]) -> () {
6895 crate::transmute::checked_transmute_store(a.val.0, dest);
6896 }
6897 #[inline(always)]
6898 fn cvt_from_bytes_f32x16(self, a: u8x64<Self>) -> f32x16<Self> {
6899 f32x16 {
6900 val: crate::transmute::checked_transmute_copy(&a.val),
6901 simd: self,
6902 }
6903 }
6904 #[inline(always)]
6905 fn cvt_to_bytes_f32x16(self, a: f32x16<Self>) -> u8x64<Self> {
6906 u8x64 {
6907 val: crate::transmute::checked_transmute_copy(&a.val),
6908 simd: self,
6909 }
6910 }
6911 #[inline(always)]
6912 fn slide_f32x16<const SHIFT: usize>(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
6913 if SHIFT >= 16usize {
6914 return b;
6915 }
6916 let result = cross_block_alignr_128x4(
6917 self,
6918 self.cvt_to_bytes_f32x16(b).val.0,
6919 self.cvt_to_bytes_f32x16(a).val.0,
6920 SHIFT * 4usize,
6921 );
6922 self.cvt_from_bytes_f32x16(u8x64 {
6923 val: crate::support::Aligned512(result),
6924 simd: self,
6925 })
6926 }
6927 #[inline(always)]
6928 fn slide_within_blocks_f32x16<const SHIFT: usize>(
6929 self,
6930 a: f32x16<Self>,
6931 b: f32x16<Self>,
6932 ) -> f32x16<Self> {
6933 let (a0, a1) = self.split_f32x16(a);
6934 let (b0, b1) = self.split_f32x16(b);
6935 self.combine_f32x8(
6936 self.slide_within_blocks_f32x8::<SHIFT>(a0, b0),
6937 self.slide_within_blocks_f32x8::<SHIFT>(a1, b1),
6938 )
6939 }
6940 #[inline(always)]
6941 fn swizzle_dyn_within_blocks_f32x16(
6942 self,
6943 a: f32x16<Self>,
6944 indices: u8x64<Self>,
6945 ) -> f32x16<Self> {
6946 let (a0, a1) = self.split_f32x16(a);
6947 let (indices0, indices1) = self.split_u8x64(indices);
6948 self.combine_f32x8(
6949 self.swizzle_dyn_within_blocks_f32x8(a0, indices0),
6950 self.swizzle_dyn_within_blocks_f32x8(a1, indices1),
6951 )
6952 }
6953 #[inline(always)]
6954 fn abs_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
6955 let (a0, a1) = self.split_f32x16(a);
6956 self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1))
6957 }
6958 #[inline(always)]
6959 fn neg_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
6960 let (a0, a1) = self.split_f32x16(a);
6961 self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1))
6962 }
6963 #[inline(always)]
6964 fn sqrt_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
6965 let (a0, a1) = self.split_f32x16(a);
6966 self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1))
6967 }
6968 #[inline(always)]
6969 fn approximate_recip_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
6970 let (a0, a1) = self.split_f32x16(a);
6971 self.combine_f32x8(
6972 self.approximate_recip_f32x8(a0),
6973 self.approximate_recip_f32x8(a1),
6974 )
6975 }
6976 #[inline(always)]
6977 fn add_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
6978 let (a0, a1) = self.split_f32x16(a);
6979 let (b0, b1) = self.split_f32x16(b);
6980 self.combine_f32x8(self.add_f32x8(a0, b0), self.add_f32x8(a1, b1))
6981 }
6982 #[inline(always)]
6983 fn sub_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
6984 let (a0, a1) = self.split_f32x16(a);
6985 let (b0, b1) = self.split_f32x16(b);
6986 self.combine_f32x8(self.sub_f32x8(a0, b0), self.sub_f32x8(a1, b1))
6987 }
6988 #[inline(always)]
6989 fn mul_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
6990 let (a0, a1) = self.split_f32x16(a);
6991 let (b0, b1) = self.split_f32x16(b);
6992 self.combine_f32x8(self.mul_f32x8(a0, b0), self.mul_f32x8(a1, b1))
6993 }
6994 #[inline(always)]
6995 fn div_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
6996 let (a0, a1) = self.split_f32x16(a);
6997 let (b0, b1) = self.split_f32x16(b);
6998 self.combine_f32x8(self.div_f32x8(a0, b0), self.div_f32x8(a1, b1))
6999 }
7000 #[inline(always)]
7001 fn copysign_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
7002 let (a0, a1) = self.split_f32x16(a);
7003 let (b0, b1) = self.split_f32x16(b);
7004 self.combine_f32x8(self.copysign_f32x8(a0, b0), self.copysign_f32x8(a1, b1))
7005 }
7006 #[inline(always)]
7007 fn simd_eq_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
7008 let (a0, a1) = self.split_f32x16(a);
7009 let (b0, b1) = self.split_f32x16(b);
7010 self.combine_mask32x8(self.simd_eq_f32x8(a0, b0), self.simd_eq_f32x8(a1, b1))
7011 }
7012 #[inline(always)]
7013 fn simd_lt_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
7014 let (a0, a1) = self.split_f32x16(a);
7015 let (b0, b1) = self.split_f32x16(b);
7016 self.combine_mask32x8(self.simd_lt_f32x8(a0, b0), self.simd_lt_f32x8(a1, b1))
7017 }
7018 #[inline(always)]
7019 fn simd_le_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
7020 let (a0, a1) = self.split_f32x16(a);
7021 let (b0, b1) = self.split_f32x16(b);
7022 self.combine_mask32x8(self.simd_le_f32x8(a0, b0), self.simd_le_f32x8(a1, b1))
7023 }
7024 #[inline(always)]
7025 fn simd_ge_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
7026 let (a0, a1) = self.split_f32x16(a);
7027 let (b0, b1) = self.split_f32x16(b);
7028 self.combine_mask32x8(self.simd_ge_f32x8(a0, b0), self.simd_ge_f32x8(a1, b1))
7029 }
7030 #[inline(always)]
7031 fn simd_gt_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
7032 let (a0, a1) = self.split_f32x16(a);
7033 let (b0, b1) = self.split_f32x16(b);
7034 self.combine_mask32x8(self.simd_gt_f32x8(a0, b0), self.simd_gt_f32x8(a1, b1))
7035 }
7036 #[inline(always)]
7037 fn zip_low_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
7038 let (a0, _) = self.split_f32x16(a);
7039 let (b0, _) = self.split_f32x16(b);
7040 self.combine_f32x8(self.zip_low_f32x8(a0, b0), self.zip_high_f32x8(a0, b0))
7041 }
7042 #[inline(always)]
7043 fn zip_high_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
7044 let (_, a1) = self.split_f32x16(a);
7045 let (_, b1) = self.split_f32x16(b);
7046 self.combine_f32x8(self.zip_low_f32x8(a1, b1), self.zip_high_f32x8(a1, b1))
7047 }
7048 #[inline(always)]
7049 fn unzip_low_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
7050 let (a0, a1) = self.split_f32x16(a);
7051 let (b0, b1) = self.split_f32x16(b);
7052 self.combine_f32x8(self.unzip_low_f32x8(a0, a1), self.unzip_low_f32x8(b0, b1))
7053 }
7054 #[inline(always)]
7055 fn unzip_high_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
7056 let (a0, a1) = self.split_f32x16(a);
7057 let (b0, b1) = self.split_f32x16(b);
7058 self.combine_f32x8(self.unzip_high_f32x8(a0, a1), self.unzip_high_f32x8(b0, b1))
7059 }
7060 #[inline(always)]
7061 fn interleave_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> (f32x16<Self>, f32x16<Self>) {
7062 let (a0, a1) = self.split_f32x16(a);
7063 let (b0, b1) = self.split_f32x16(b);
7064 let lo_lo = self.zip_low_f32x8(a0, b0);
7065 let lo_hi = self.zip_high_f32x8(a0, b0);
7066 let hi_lo = self.zip_low_f32x8(a1, b1);
7067 let hi_hi = self.zip_high_f32x8(a1, b1);
7068 (
7069 self.combine_f32x8(lo_lo, lo_hi),
7070 self.combine_f32x8(hi_lo, hi_hi),
7071 )
7072 }
7073 #[inline(always)]
7074 fn deinterleave_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> (f32x16<Self>, f32x16<Self>) {
7075 let (a0, a1) = self.split_f32x16(a);
7076 let (b0, b1) = self.split_f32x16(b);
7077 let lo_even = self.unzip_low_f32x8(a0, a1);
7078 let lo_odd = self.unzip_high_f32x8(a0, a1);
7079 let hi_even = self.unzip_low_f32x8(b0, b1);
7080 let hi_odd = self.unzip_high_f32x8(b0, b1);
7081 (
7082 self.combine_f32x8(lo_even, hi_even),
7083 self.combine_f32x8(lo_odd, hi_odd),
7084 )
7085 }
7086 #[inline(always)]
7087 fn max_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
7088 let (a0, a1) = self.split_f32x16(a);
7089 let (b0, b1) = self.split_f32x16(b);
7090 self.combine_f32x8(self.max_f32x8(a0, b0), self.max_f32x8(a1, b1))
7091 }
7092 #[inline(always)]
7093 fn min_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
7094 let (a0, a1) = self.split_f32x16(a);
7095 let (b0, b1) = self.split_f32x16(b);
7096 self.combine_f32x8(self.min_f32x8(a0, b0), self.min_f32x8(a1, b1))
7097 }
7098 #[inline(always)]
7099 fn max_precise_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
7100 let (a0, a1) = self.split_f32x16(a);
7101 let (b0, b1) = self.split_f32x16(b);
7102 self.combine_f32x8(
7103 self.max_precise_f32x8(a0, b0),
7104 self.max_precise_f32x8(a1, b1),
7105 )
7106 }
7107 #[inline(always)]
7108 fn min_precise_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
7109 let (a0, a1) = self.split_f32x16(a);
7110 let (b0, b1) = self.split_f32x16(b);
7111 self.combine_f32x8(
7112 self.min_precise_f32x8(a0, b0),
7113 self.min_precise_f32x8(a1, b1),
7114 )
7115 }
7116 #[inline(always)]
7117 fn mul_add_f32x16(self, a: f32x16<Self>, b: f32x16<Self>, c: f32x16<Self>) -> f32x16<Self> {
7118 let (a0, a1) = self.split_f32x16(a);
7119 let (b0, b1) = self.split_f32x16(b);
7120 let (c0, c1) = self.split_f32x16(c);
7121 self.combine_f32x8(
7122 self.mul_add_f32x8(a0, b0, c0),
7123 self.mul_add_f32x8(a1, b1, c1),
7124 )
7125 }
7126 #[inline(always)]
7127 fn mul_sub_f32x16(self, a: f32x16<Self>, b: f32x16<Self>, c: f32x16<Self>) -> f32x16<Self> {
7128 let (a0, a1) = self.split_f32x16(a);
7129 let (b0, b1) = self.split_f32x16(b);
7130 let (c0, c1) = self.split_f32x16(c);
7131 self.combine_f32x8(
7132 self.mul_sub_f32x8(a0, b0, c0),
7133 self.mul_sub_f32x8(a1, b1, c1),
7134 )
7135 }
7136 #[inline(always)]
7137 fn floor_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
7138 let (a0, a1) = self.split_f32x16(a);
7139 self.combine_f32x8(self.floor_f32x8(a0), self.floor_f32x8(a1))
7140 }
7141 #[inline(always)]
7142 fn ceil_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
7143 let (a0, a1) = self.split_f32x16(a);
7144 self.combine_f32x8(self.ceil_f32x8(a0), self.ceil_f32x8(a1))
7145 }
7146 #[inline(always)]
7147 fn round_ties_even_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
7148 let (a0, a1) = self.split_f32x16(a);
7149 self.combine_f32x8(
7150 self.round_ties_even_f32x8(a0),
7151 self.round_ties_even_f32x8(a1),
7152 )
7153 }
7154 #[inline(always)]
7155 fn fract_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
7156 let (a0, a1) = self.split_f32x16(a);
7157 self.combine_f32x8(self.fract_f32x8(a0), self.fract_f32x8(a1))
7158 }
7159 #[inline(always)]
7160 fn trunc_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
7161 let (a0, a1) = self.split_f32x16(a);
7162 self.combine_f32x8(self.trunc_f32x8(a0), self.trunc_f32x8(a1))
7163 }
7164 #[inline(always)]
7165 fn select_f32x16(self, a: mask32x16<Self>, b: f32x16<Self>, c: f32x16<Self>) -> f32x16<Self> {
7166 let (a0, a1) = self.split_mask32x16(a);
7167 let (b0, b1) = self.split_f32x16(b);
7168 let (c0, c1) = self.split_f32x16(c);
7169 self.combine_f32x8(self.select_f32x8(a0, b0, c0), self.select_f32x8(a1, b1, c1))
7170 }
7171 #[inline(always)]
7172 fn split_f32x16(self, a: f32x16<Self>) -> (f32x8<Self>, f32x8<Self>) {
7173 (
7174 f32x8 {
7175 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
7176 simd: self,
7177 },
7178 f32x8 {
7179 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
7180 simd: self,
7181 },
7182 )
7183 }
7184 #[inline(always)]
7185 fn reinterpret_f64_f32x16(self, a: f32x16<Self>) -> f64x8<Self> {
7186 let (a0, a1) = self.split_f32x16(a);
7187 self.combine_f64x4(
7188 self.reinterpret_f64_f32x8(a0),
7189 self.reinterpret_f64_f32x8(a1),
7190 )
7191 }
7192 #[inline(always)]
7193 fn reinterpret_i32_f32x16(self, a: f32x16<Self>) -> i32x16<Self> {
7194 let (a0, a1) = self.split_f32x16(a);
7195 self.combine_i32x8(
7196 self.reinterpret_i32_f32x8(a0),
7197 self.reinterpret_i32_f32x8(a1),
7198 )
7199 }
7200 #[inline(always)]
7201 fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16<Self> {
7202 crate::kernel!(
7203 #[inline(always)]
7204 fn kernel(token: Sse4_2, src: &[f32; 16usize]) -> f32x16<Sse4_2> {
7205 let (chunks, []) = src.as_chunks::<4usize>() else {
7206 unreachable!()
7207 };
7208 let v0: __m128 =
7209 crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]);
7210 let v1: __m128 =
7211 crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]);
7212 let v2: __m128 =
7213 crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]);
7214 let v3: __m128 =
7215 crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]);
7216 let tmp0 = _mm_unpacklo_ps(v0, v1);
7217 let tmp1 = _mm_unpackhi_ps(v0, v1);
7218 let tmp2 = _mm_unpacklo_ps(v2, v3);
7219 let tmp3 = _mm_unpackhi_ps(v2, v3);
7220 let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
7221 let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
7222 let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
7223 let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
7224 token.combine_f32x8(
7225 token.combine_f32x4(out0.simd_into(token), out1.simd_into(token)),
7226 token.combine_f32x4(out2.simd_into(token), out3.simd_into(token)),
7227 )
7228 }
7229 );
7230 kernel(self, src)
7231 }
7232 #[inline(always)]
7233 fn store_interleaved_128_f32x16(self, a: f32x16<Self>, dest: &mut [f32; 16usize]) -> () {
7234 crate::kernel!(
7235 #[inline(always)]
7236 fn kernel(token: Sse4_2, a: f32x16<Sse4_2>, dest: &mut [f32; 16usize]) -> () {
7237 let (v01, v23) = token.split_f32x16(a);
7238 let (v0, v1) = token.split_f32x8(v01);
7239 let (v2, v3) = token.split_f32x8(v23);
7240 let v0 = v0.into();
7241 let v1 = v1.into();
7242 let v2 = v2.into();
7243 let v3 = v3.into();
7244 let tmp0 = _mm_unpacklo_ps(v0, v1);
7245 let tmp1 = _mm_unpackhi_ps(v0, v1);
7246 let tmp2 = _mm_unpacklo_ps(v2, v3);
7247 let tmp3 = _mm_unpackhi_ps(v2, v3);
7248 let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
7249 let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
7250 let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
7251 let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
7252 let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
7253 unreachable!()
7254 };
7255 crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
7256 out0,
7257 &mut chunks[0],
7258 );
7259 crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
7260 out1,
7261 &mut chunks[1],
7262 );
7263 crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
7264 out2,
7265 &mut chunks[2],
7266 );
7267 crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
7268 out3,
7269 &mut chunks[3],
7270 );
7271 }
7272 );
7273 kernel(self, a, dest);
7274 }
7275 #[inline(always)]
7276 fn reinterpret_u8_f32x16(self, a: f32x16<Self>) -> u8x64<Self> {
7277 let (a0, a1) = self.split_f32x16(a);
7278 self.combine_u8x32(self.reinterpret_u8_f32x8(a0), self.reinterpret_u8_f32x8(a1))
7279 }
7280 #[inline(always)]
7281 fn reinterpret_u32_f32x16(self, a: f32x16<Self>) -> u32x16<Self> {
7282 let (a0, a1) = self.split_f32x16(a);
7283 self.combine_u32x8(
7284 self.reinterpret_u32_f32x8(a0),
7285 self.reinterpret_u32_f32x8(a1),
7286 )
7287 }
7288 #[inline(always)]
7289 fn cvt_u32_f32x16(self, a: f32x16<Self>) -> u32x16<Self> {
7290 let (a0, a1) = self.split_f32x16(a);
7291 self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1))
7292 }
7293 #[inline(always)]
7294 fn cvt_u32_precise_f32x16(self, a: f32x16<Self>) -> u32x16<Self> {
7295 let (a0, a1) = self.split_f32x16(a);
7296 self.combine_u32x8(
7297 self.cvt_u32_precise_f32x8(a0),
7298 self.cvt_u32_precise_f32x8(a1),
7299 )
7300 }
7301 #[inline(always)]
7302 fn cvt_i32_f32x16(self, a: f32x16<Self>) -> i32x16<Self> {
7303 let (a0, a1) = self.split_f32x16(a);
7304 self.combine_i32x8(self.cvt_i32_f32x8(a0), self.cvt_i32_f32x8(a1))
7305 }
7306 #[inline(always)]
7307 fn cvt_i32_precise_f32x16(self, a: f32x16<Self>) -> i32x16<Self> {
7308 let (a0, a1) = self.split_f32x16(a);
7309 self.combine_i32x8(
7310 self.cvt_i32_precise_f32x8(a0),
7311 self.cvt_i32_precise_f32x8(a1),
7312 )
7313 }
7314 #[inline(always)]
7315 fn splat_i8x64(self, val: i8) -> i8x64<Self> {
7316 let half = self.splat_i8x32(val);
7317 self.combine_i8x32(half, half)
7318 }
7319 #[inline(always)]
7320 fn load_array_i8x64(self, val: [i8; 64usize]) -> i8x64<Self> {
7321 i8x64 {
7322 val: crate::transmute::checked_transmute_copy(&val),
7323 simd: self,
7324 }
7325 }
7326 #[inline(always)]
7327 fn load_array_ref_i8x64(self, val: &[i8; 64usize]) -> i8x64<Self> {
7328 i8x64 {
7329 val: crate::transmute::checked_transmute_copy(val),
7330 simd: self,
7331 }
7332 }
7333 #[inline(always)]
7334 fn as_array_i8x64(self, a: i8x64<Self>) -> [i8; 64usize] {
7335 crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i8; 64usize]>(&a.val.0)
7336 }
7337 #[inline(always)]
7338 fn as_array_ref_i8x64(self, a: &i8x64<Self>) -> &[i8; 64usize] {
7339 crate::transmute::checked_cast_ref::<[__m128i; 4usize], [i8; 64usize]>(&a.val.0)
7340 }
7341 #[inline(always)]
7342 fn as_array_mut_i8x64(self, a: &mut i8x64<Self>) -> &mut [i8; 64usize] {
7343 crate::transmute::checked_cast_mut::<[__m128i; 4usize], [i8; 64usize]>(&mut a.val.0)
7344 }
7345 #[inline(always)]
7346 fn store_array_i8x64(self, a: i8x64<Self>, dest: &mut [i8; 64usize]) -> () {
7347 crate::transmute::checked_transmute_store(a.val.0, dest);
7348 }
7349 #[inline(always)]
7350 fn cvt_from_bytes_i8x64(self, a: u8x64<Self>) -> i8x64<Self> {
7351 i8x64 {
7352 val: crate::transmute::checked_transmute_copy(&a.val),
7353 simd: self,
7354 }
7355 }
7356 #[inline(always)]
7357 fn cvt_to_bytes_i8x64(self, a: i8x64<Self>) -> u8x64<Self> {
7358 u8x64 {
7359 val: crate::transmute::checked_transmute_copy(&a.val),
7360 simd: self,
7361 }
7362 }
7363 #[inline(always)]
7364 fn slide_i8x64<const SHIFT: usize>(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7365 if SHIFT >= 64usize {
7366 return b;
7367 }
7368 let result = cross_block_alignr_128x4(
7369 self,
7370 self.cvt_to_bytes_i8x64(b).val.0,
7371 self.cvt_to_bytes_i8x64(a).val.0,
7372 SHIFT,
7373 );
7374 self.cvt_from_bytes_i8x64(u8x64 {
7375 val: crate::support::Aligned512(result),
7376 simd: self,
7377 })
7378 }
7379 #[inline(always)]
7380 fn slide_within_blocks_i8x64<const SHIFT: usize>(
7381 self,
7382 a: i8x64<Self>,
7383 b: i8x64<Self>,
7384 ) -> i8x64<Self> {
7385 let (a0, a1) = self.split_i8x64(a);
7386 let (b0, b1) = self.split_i8x64(b);
7387 self.combine_i8x32(
7388 self.slide_within_blocks_i8x32::<SHIFT>(a0, b0),
7389 self.slide_within_blocks_i8x32::<SHIFT>(a1, b1),
7390 )
7391 }
7392 #[inline(always)]
7393 fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64<Self>, indices: u8x64<Self>) -> i8x64<Self> {
7394 let (a0, a1) = self.split_i8x64(a);
7395 let (indices0, indices1) = self.split_u8x64(indices);
7396 self.combine_i8x32(
7397 self.swizzle_dyn_within_blocks_i8x32(a0, indices0),
7398 self.swizzle_dyn_within_blocks_i8x32(a1, indices1),
7399 )
7400 }
7401 #[inline(always)]
7402 fn add_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7403 let (a0, a1) = self.split_i8x64(a);
7404 let (b0, b1) = self.split_i8x64(b);
7405 self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1))
7406 }
7407 #[inline(always)]
7408 fn sub_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7409 let (a0, a1) = self.split_i8x64(a);
7410 let (b0, b1) = self.split_i8x64(b);
7411 self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1))
7412 }
7413 #[inline(always)]
7414 fn mul_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7415 let (a0, a1) = self.split_i8x64(a);
7416 let (b0, b1) = self.split_i8x64(b);
7417 self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1))
7418 }
7419 #[inline(always)]
7420 fn and_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7421 let (a0, a1) = self.split_i8x64(a);
7422 let (b0, b1) = self.split_i8x64(b);
7423 self.combine_i8x32(self.and_i8x32(a0, b0), self.and_i8x32(a1, b1))
7424 }
7425 #[inline(always)]
7426 fn or_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7427 let (a0, a1) = self.split_i8x64(a);
7428 let (b0, b1) = self.split_i8x64(b);
7429 self.combine_i8x32(self.or_i8x32(a0, b0), self.or_i8x32(a1, b1))
7430 }
7431 #[inline(always)]
7432 fn xor_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7433 let (a0, a1) = self.split_i8x64(a);
7434 let (b0, b1) = self.split_i8x64(b);
7435 self.combine_i8x32(self.xor_i8x32(a0, b0), self.xor_i8x32(a1, b1))
7436 }
7437 #[inline(always)]
7438 fn not_i8x64(self, a: i8x64<Self>) -> i8x64<Self> {
7439 let (a0, a1) = self.split_i8x64(a);
7440 self.combine_i8x32(self.not_i8x32(a0), self.not_i8x32(a1))
7441 }
7442 #[inline(always)]
7443 fn shl_i8x64(self, a: i8x64<Self>, shift: u32) -> i8x64<Self> {
7444 let (a0, a1) = self.split_i8x64(a);
7445 self.combine_i8x32(self.shl_i8x32(a0, shift), self.shl_i8x32(a1, shift))
7446 }
7447 #[inline(always)]
7448 fn shlv_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7449 let (a0, a1) = self.split_i8x64(a);
7450 let (b0, b1) = self.split_i8x64(b);
7451 self.combine_i8x32(self.shlv_i8x32(a0, b0), self.shlv_i8x32(a1, b1))
7452 }
7453 #[inline(always)]
7454 fn shr_i8x64(self, a: i8x64<Self>, shift: u32) -> i8x64<Self> {
7455 let (a0, a1) = self.split_i8x64(a);
7456 self.combine_i8x32(self.shr_i8x32(a0, shift), self.shr_i8x32(a1, shift))
7457 }
7458 #[inline(always)]
7459 fn shrv_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7460 let (a0, a1) = self.split_i8x64(a);
7461 let (b0, b1) = self.split_i8x64(b);
7462 self.combine_i8x32(self.shrv_i8x32(a0, b0), self.shrv_i8x32(a1, b1))
7463 }
7464 #[inline(always)]
7465 fn simd_eq_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
7466 let (a0, a1) = self.split_i8x64(a);
7467 let (b0, b1) = self.split_i8x64(b);
7468 self.combine_mask8x32(self.simd_eq_i8x32(a0, b0), self.simd_eq_i8x32(a1, b1))
7469 }
7470 #[inline(always)]
7471 fn simd_lt_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
7472 let (a0, a1) = self.split_i8x64(a);
7473 let (b0, b1) = self.split_i8x64(b);
7474 self.combine_mask8x32(self.simd_lt_i8x32(a0, b0), self.simd_lt_i8x32(a1, b1))
7475 }
7476 #[inline(always)]
7477 fn simd_le_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
7478 let (a0, a1) = self.split_i8x64(a);
7479 let (b0, b1) = self.split_i8x64(b);
7480 self.combine_mask8x32(self.simd_le_i8x32(a0, b0), self.simd_le_i8x32(a1, b1))
7481 }
7482 #[inline(always)]
7483 fn simd_ge_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
7484 let (a0, a1) = self.split_i8x64(a);
7485 let (b0, b1) = self.split_i8x64(b);
7486 self.combine_mask8x32(self.simd_ge_i8x32(a0, b0), self.simd_ge_i8x32(a1, b1))
7487 }
7488 #[inline(always)]
7489 fn simd_gt_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
7490 let (a0, a1) = self.split_i8x64(a);
7491 let (b0, b1) = self.split_i8x64(b);
7492 self.combine_mask8x32(self.simd_gt_i8x32(a0, b0), self.simd_gt_i8x32(a1, b1))
7493 }
7494 #[inline(always)]
7495 fn zip_low_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7496 let (a0, _) = self.split_i8x64(a);
7497 let (b0, _) = self.split_i8x64(b);
7498 self.combine_i8x32(self.zip_low_i8x32(a0, b0), self.zip_high_i8x32(a0, b0))
7499 }
7500 #[inline(always)]
7501 fn zip_high_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7502 let (_, a1) = self.split_i8x64(a);
7503 let (_, b1) = self.split_i8x64(b);
7504 self.combine_i8x32(self.zip_low_i8x32(a1, b1), self.zip_high_i8x32(a1, b1))
7505 }
7506 #[inline(always)]
7507 fn unzip_low_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7508 let (a0, a1) = self.split_i8x64(a);
7509 let (b0, b1) = self.split_i8x64(b);
7510 self.combine_i8x32(self.unzip_low_i8x32(a0, a1), self.unzip_low_i8x32(b0, b1))
7511 }
7512 #[inline(always)]
7513 fn unzip_high_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7514 let (a0, a1) = self.split_i8x64(a);
7515 let (b0, b1) = self.split_i8x64(b);
7516 self.combine_i8x32(self.unzip_high_i8x32(a0, a1), self.unzip_high_i8x32(b0, b1))
7517 }
7518 #[inline(always)]
7519 fn interleave_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> (i8x64<Self>, i8x64<Self>) {
7520 let (a0, a1) = self.split_i8x64(a);
7521 let (b0, b1) = self.split_i8x64(b);
7522 let lo_lo = self.zip_low_i8x32(a0, b0);
7523 let lo_hi = self.zip_high_i8x32(a0, b0);
7524 let hi_lo = self.zip_low_i8x32(a1, b1);
7525 let hi_hi = self.zip_high_i8x32(a1, b1);
7526 (
7527 self.combine_i8x32(lo_lo, lo_hi),
7528 self.combine_i8x32(hi_lo, hi_hi),
7529 )
7530 }
7531 #[inline(always)]
7532 fn deinterleave_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> (i8x64<Self>, i8x64<Self>) {
7533 let (a0, a1) = self.split_i8x64(a);
7534 let (b0, b1) = self.split_i8x64(b);
7535 let lo_even = self.unzip_low_i8x32(a0, a1);
7536 let lo_odd = self.unzip_high_i8x32(a0, a1);
7537 let hi_even = self.unzip_low_i8x32(b0, b1);
7538 let hi_odd = self.unzip_high_i8x32(b0, b1);
7539 (
7540 self.combine_i8x32(lo_even, hi_even),
7541 self.combine_i8x32(lo_odd, hi_odd),
7542 )
7543 }
7544 #[inline(always)]
7545 fn select_i8x64(self, a: mask8x64<Self>, b: i8x64<Self>, c: i8x64<Self>) -> i8x64<Self> {
7546 let (a0, a1) = self.split_mask8x64(a);
7547 let (b0, b1) = self.split_i8x64(b);
7548 let (c0, c1) = self.split_i8x64(c);
7549 self.combine_i8x32(self.select_i8x32(a0, b0, c0), self.select_i8x32(a1, b1, c1))
7550 }
7551 #[inline(always)]
7552 fn min_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7553 let (a0, a1) = self.split_i8x64(a);
7554 let (b0, b1) = self.split_i8x64(b);
7555 self.combine_i8x32(self.min_i8x32(a0, b0), self.min_i8x32(a1, b1))
7556 }
7557 #[inline(always)]
7558 fn max_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
7559 let (a0, a1) = self.split_i8x64(a);
7560 let (b0, b1) = self.split_i8x64(b);
7561 self.combine_i8x32(self.max_i8x32(a0, b0), self.max_i8x32(a1, b1))
7562 }
7563 #[inline(always)]
7564 fn split_i8x64(self, a: i8x64<Self>) -> (i8x32<Self>, i8x32<Self>) {
7565 (
7566 i8x32 {
7567 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
7568 simd: self,
7569 },
7570 i8x32 {
7571 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
7572 simd: self,
7573 },
7574 )
7575 }
7576 #[inline(always)]
7577 fn neg_i8x64(self, a: i8x64<Self>) -> i8x64<Self> {
7578 let (a0, a1) = self.split_i8x64(a);
7579 self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1))
7580 }
7581 #[inline(always)]
7582 fn reinterpret_u8_i8x64(self, a: i8x64<Self>) -> u8x64<Self> {
7583 let (a0, a1) = self.split_i8x64(a);
7584 self.combine_u8x32(self.reinterpret_u8_i8x32(a0), self.reinterpret_u8_i8x32(a1))
7585 }
7586 #[inline(always)]
7587 fn reinterpret_u32_i8x64(self, a: i8x64<Self>) -> u32x16<Self> {
7588 let (a0, a1) = self.split_i8x64(a);
7589 self.combine_u32x8(
7590 self.reinterpret_u32_i8x32(a0),
7591 self.reinterpret_u32_i8x32(a1),
7592 )
7593 }
7594 #[inline(always)]
7595 fn splat_u8x64(self, val: u8) -> u8x64<Self> {
7596 let half = self.splat_u8x32(val);
7597 self.combine_u8x32(half, half)
7598 }
7599 #[inline(always)]
7600 fn load_array_u8x64(self, val: [u8; 64usize]) -> u8x64<Self> {
7601 u8x64 {
7602 val: crate::transmute::checked_transmute_copy(&val),
7603 simd: self,
7604 }
7605 }
7606 #[inline(always)]
7607 fn load_array_ref_u8x64(self, val: &[u8; 64usize]) -> u8x64<Self> {
7608 u8x64 {
7609 val: crate::transmute::checked_transmute_copy(val),
7610 simd: self,
7611 }
7612 }
7613 #[inline(always)]
7614 fn as_array_u8x64(self, a: u8x64<Self>) -> [u8; 64usize] {
7615 crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [u8; 64usize]>(&a.val.0)
7616 }
7617 #[inline(always)]
7618 fn as_array_ref_u8x64(self, a: &u8x64<Self>) -> &[u8; 64usize] {
7619 crate::transmute::checked_cast_ref::<[__m128i; 4usize], [u8; 64usize]>(&a.val.0)
7620 }
7621 #[inline(always)]
7622 fn as_array_mut_u8x64(self, a: &mut u8x64<Self>) -> &mut [u8; 64usize] {
7623 crate::transmute::checked_cast_mut::<[__m128i; 4usize], [u8; 64usize]>(&mut a.val.0)
7624 }
7625 #[inline(always)]
7626 fn store_array_u8x64(self, a: u8x64<Self>, dest: &mut [u8; 64usize]) -> () {
7627 crate::transmute::checked_transmute_store(a.val.0, dest);
7628 }
7629 #[inline(always)]
7630 fn cvt_from_bytes_u8x64(self, a: u8x64<Self>) -> u8x64<Self> {
7631 u8x64 {
7632 val: crate::transmute::checked_transmute_copy(&a.val),
7633 simd: self,
7634 }
7635 }
7636 #[inline(always)]
7637 fn cvt_to_bytes_u8x64(self, a: u8x64<Self>) -> u8x64<Self> {
7638 u8x64 {
7639 val: crate::transmute::checked_transmute_copy(&a.val),
7640 simd: self,
7641 }
7642 }
7643 #[inline(always)]
7644 fn slide_u8x64<const SHIFT: usize>(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7645 if SHIFT >= 64usize {
7646 return b;
7647 }
7648 let result = cross_block_alignr_128x4(
7649 self,
7650 self.cvt_to_bytes_u8x64(b).val.0,
7651 self.cvt_to_bytes_u8x64(a).val.0,
7652 SHIFT,
7653 );
7654 self.cvt_from_bytes_u8x64(u8x64 {
7655 val: crate::support::Aligned512(result),
7656 simd: self,
7657 })
7658 }
7659 #[inline(always)]
7660 fn slide_within_blocks_u8x64<const SHIFT: usize>(
7661 self,
7662 a: u8x64<Self>,
7663 b: u8x64<Self>,
7664 ) -> u8x64<Self> {
7665 let (a0, a1) = self.split_u8x64(a);
7666 let (b0, b1) = self.split_u8x64(b);
7667 self.combine_u8x32(
7668 self.slide_within_blocks_u8x32::<SHIFT>(a0, b0),
7669 self.slide_within_blocks_u8x32::<SHIFT>(a1, b1),
7670 )
7671 }
7672 #[inline(always)]
7673 fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
7674 let (a0, a1) = self.split_u8x64(a);
7675 let (indices0, indices1) = self.split_u8x64(indices);
7676 self.combine_u8x32(
7677 self.swizzle_dyn_within_blocks_u8x32(a0, indices0),
7678 self.swizzle_dyn_within_blocks_u8x32(a1, indices1),
7679 )
7680 }
7681 #[inline(always)]
7682 fn add_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7683 let (a0, a1) = self.split_u8x64(a);
7684 let (b0, b1) = self.split_u8x64(b);
7685 self.combine_u8x32(self.add_u8x32(a0, b0), self.add_u8x32(a1, b1))
7686 }
7687 #[inline(always)]
7688 fn sub_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7689 let (a0, a1) = self.split_u8x64(a);
7690 let (b0, b1) = self.split_u8x64(b);
7691 self.combine_u8x32(self.sub_u8x32(a0, b0), self.sub_u8x32(a1, b1))
7692 }
7693 #[inline(always)]
7694 fn mul_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7695 let (a0, a1) = self.split_u8x64(a);
7696 let (b0, b1) = self.split_u8x64(b);
7697 self.combine_u8x32(self.mul_u8x32(a0, b0), self.mul_u8x32(a1, b1))
7698 }
7699 #[inline(always)]
7700 fn and_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7701 let (a0, a1) = self.split_u8x64(a);
7702 let (b0, b1) = self.split_u8x64(b);
7703 self.combine_u8x32(self.and_u8x32(a0, b0), self.and_u8x32(a1, b1))
7704 }
7705 #[inline(always)]
7706 fn or_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7707 let (a0, a1) = self.split_u8x64(a);
7708 let (b0, b1) = self.split_u8x64(b);
7709 self.combine_u8x32(self.or_u8x32(a0, b0), self.or_u8x32(a1, b1))
7710 }
7711 #[inline(always)]
7712 fn xor_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7713 let (a0, a1) = self.split_u8x64(a);
7714 let (b0, b1) = self.split_u8x64(b);
7715 self.combine_u8x32(self.xor_u8x32(a0, b0), self.xor_u8x32(a1, b1))
7716 }
7717 #[inline(always)]
7718 fn not_u8x64(self, a: u8x64<Self>) -> u8x64<Self> {
7719 let (a0, a1) = self.split_u8x64(a);
7720 self.combine_u8x32(self.not_u8x32(a0), self.not_u8x32(a1))
7721 }
7722 #[inline(always)]
7723 fn shl_u8x64(self, a: u8x64<Self>, shift: u32) -> u8x64<Self> {
7724 let (a0, a1) = self.split_u8x64(a);
7725 self.combine_u8x32(self.shl_u8x32(a0, shift), self.shl_u8x32(a1, shift))
7726 }
7727 #[inline(always)]
7728 fn shlv_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7729 let (a0, a1) = self.split_u8x64(a);
7730 let (b0, b1) = self.split_u8x64(b);
7731 self.combine_u8x32(self.shlv_u8x32(a0, b0), self.shlv_u8x32(a1, b1))
7732 }
7733 #[inline(always)]
7734 fn shr_u8x64(self, a: u8x64<Self>, shift: u32) -> u8x64<Self> {
7735 let (a0, a1) = self.split_u8x64(a);
7736 self.combine_u8x32(self.shr_u8x32(a0, shift), self.shr_u8x32(a1, shift))
7737 }
7738 #[inline(always)]
7739 fn shrv_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7740 let (a0, a1) = self.split_u8x64(a);
7741 let (b0, b1) = self.split_u8x64(b);
7742 self.combine_u8x32(self.shrv_u8x32(a0, b0), self.shrv_u8x32(a1, b1))
7743 }
7744 #[inline(always)]
7745 fn simd_eq_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
7746 let (a0, a1) = self.split_u8x64(a);
7747 let (b0, b1) = self.split_u8x64(b);
7748 self.combine_mask8x32(self.simd_eq_u8x32(a0, b0), self.simd_eq_u8x32(a1, b1))
7749 }
7750 #[inline(always)]
7751 fn simd_lt_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
7752 let (a0, a1) = self.split_u8x64(a);
7753 let (b0, b1) = self.split_u8x64(b);
7754 self.combine_mask8x32(self.simd_lt_u8x32(a0, b0), self.simd_lt_u8x32(a1, b1))
7755 }
7756 #[inline(always)]
7757 fn simd_le_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
7758 let (a0, a1) = self.split_u8x64(a);
7759 let (b0, b1) = self.split_u8x64(b);
7760 self.combine_mask8x32(self.simd_le_u8x32(a0, b0), self.simd_le_u8x32(a1, b1))
7761 }
7762 #[inline(always)]
7763 fn simd_ge_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
7764 let (a0, a1) = self.split_u8x64(a);
7765 let (b0, b1) = self.split_u8x64(b);
7766 self.combine_mask8x32(self.simd_ge_u8x32(a0, b0), self.simd_ge_u8x32(a1, b1))
7767 }
7768 #[inline(always)]
7769 fn simd_gt_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
7770 let (a0, a1) = self.split_u8x64(a);
7771 let (b0, b1) = self.split_u8x64(b);
7772 self.combine_mask8x32(self.simd_gt_u8x32(a0, b0), self.simd_gt_u8x32(a1, b1))
7773 }
7774 #[inline(always)]
7775 fn zip_low_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7776 let (a0, _) = self.split_u8x64(a);
7777 let (b0, _) = self.split_u8x64(b);
7778 self.combine_u8x32(self.zip_low_u8x32(a0, b0), self.zip_high_u8x32(a0, b0))
7779 }
7780 #[inline(always)]
7781 fn zip_high_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7782 let (_, a1) = self.split_u8x64(a);
7783 let (_, b1) = self.split_u8x64(b);
7784 self.combine_u8x32(self.zip_low_u8x32(a1, b1), self.zip_high_u8x32(a1, b1))
7785 }
7786 #[inline(always)]
7787 fn unzip_low_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7788 let (a0, a1) = self.split_u8x64(a);
7789 let (b0, b1) = self.split_u8x64(b);
7790 self.combine_u8x32(self.unzip_low_u8x32(a0, a1), self.unzip_low_u8x32(b0, b1))
7791 }
7792 #[inline(always)]
7793 fn unzip_high_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7794 let (a0, a1) = self.split_u8x64(a);
7795 let (b0, b1) = self.split_u8x64(b);
7796 self.combine_u8x32(self.unzip_high_u8x32(a0, a1), self.unzip_high_u8x32(b0, b1))
7797 }
7798 #[inline(always)]
7799 fn interleave_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> (u8x64<Self>, u8x64<Self>) {
7800 let (a0, a1) = self.split_u8x64(a);
7801 let (b0, b1) = self.split_u8x64(b);
7802 let lo_lo = self.zip_low_u8x32(a0, b0);
7803 let lo_hi = self.zip_high_u8x32(a0, b0);
7804 let hi_lo = self.zip_low_u8x32(a1, b1);
7805 let hi_hi = self.zip_high_u8x32(a1, b1);
7806 (
7807 self.combine_u8x32(lo_lo, lo_hi),
7808 self.combine_u8x32(hi_lo, hi_hi),
7809 )
7810 }
7811 #[inline(always)]
7812 fn deinterleave_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> (u8x64<Self>, u8x64<Self>) {
7813 let (a0, a1) = self.split_u8x64(a);
7814 let (b0, b1) = self.split_u8x64(b);
7815 let lo_even = self.unzip_low_u8x32(a0, a1);
7816 let lo_odd = self.unzip_high_u8x32(a0, a1);
7817 let hi_even = self.unzip_low_u8x32(b0, b1);
7818 let hi_odd = self.unzip_high_u8x32(b0, b1);
7819 (
7820 self.combine_u8x32(lo_even, hi_even),
7821 self.combine_u8x32(lo_odd, hi_odd),
7822 )
7823 }
7824 #[inline(always)]
7825 fn select_u8x64(self, a: mask8x64<Self>, b: u8x64<Self>, c: u8x64<Self>) -> u8x64<Self> {
7826 let (a0, a1) = self.split_mask8x64(a);
7827 let (b0, b1) = self.split_u8x64(b);
7828 let (c0, c1) = self.split_u8x64(c);
7829 self.combine_u8x32(self.select_u8x32(a0, b0, c0), self.select_u8x32(a1, b1, c1))
7830 }
7831 #[inline(always)]
7832 fn min_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7833 let (a0, a1) = self.split_u8x64(a);
7834 let (b0, b1) = self.split_u8x64(b);
7835 self.combine_u8x32(self.min_u8x32(a0, b0), self.min_u8x32(a1, b1))
7836 }
7837 #[inline(always)]
7838 fn max_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
7839 let (a0, a1) = self.split_u8x64(a);
7840 let (b0, b1) = self.split_u8x64(b);
7841 self.combine_u8x32(self.max_u8x32(a0, b0), self.max_u8x32(a1, b1))
7842 }
7843 #[inline(always)]
7844 fn split_u8x64(self, a: u8x64<Self>) -> (u8x32<Self>, u8x32<Self>) {
7845 (
7846 u8x32 {
7847 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
7848 simd: self,
7849 },
7850 u8x32 {
7851 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
7852 simd: self,
7853 },
7854 )
7855 }
7856 #[inline(always)]
7857 fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64<Self> {
7858 crate::kernel!(
7859 #[inline(always)]
7860 fn kernel(token: Sse4_2, src: &[u8; 64usize]) -> u8x64<Sse4_2> {
7861 let (chunks, []) = src.as_chunks::<16usize>() else {
7862 unreachable!()
7863 };
7864 let v0: __m128i =
7865 crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]);
7866 let v1: __m128i =
7867 crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]);
7868 let v2: __m128i =
7869 crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]);
7870 let v3: __m128i =
7871 crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]);
7872 let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
7873 let v0 = _mm_shuffle_epi8(v0, mask);
7874 let v1 = _mm_shuffle_epi8(v1, mask);
7875 let v2 = _mm_shuffle_epi8(v2, mask);
7876 let v3 = _mm_shuffle_epi8(v3, mask);
7877 let tmp0 = _mm_unpacklo_epi32(v0, v1);
7878 let tmp1 = _mm_unpackhi_epi32(v0, v1);
7879 let tmp2 = _mm_unpacklo_epi32(v2, v3);
7880 let tmp3 = _mm_unpackhi_epi32(v2, v3);
7881 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
7882 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
7883 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
7884 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
7885 token.combine_u8x32(
7886 token.combine_u8x16(out0.simd_into(token), out1.simd_into(token)),
7887 token.combine_u8x16(out2.simd_into(token), out3.simd_into(token)),
7888 )
7889 }
7890 );
7891 kernel(self, src)
7892 }
7893 #[inline(always)]
7894 fn store_interleaved_128_u8x64(self, a: u8x64<Self>, dest: &mut [u8; 64usize]) -> () {
7895 crate::kernel!(
7896 #[inline(always)]
7897 fn kernel(token: Sse4_2, a: u8x64<Sse4_2>, dest: &mut [u8; 64usize]) -> () {
7898 let (v01, v23) = token.split_u8x64(a);
7899 let (v0, v1) = token.split_u8x32(v01);
7900 let (v2, v3) = token.split_u8x32(v23);
7901 let v0 = v0.into();
7902 let v1 = v1.into();
7903 let v2 = v2.into();
7904 let v3 = v3.into();
7905 let tmp0 = _mm_unpacklo_epi32(v0, v1);
7906 let tmp1 = _mm_unpackhi_epi32(v0, v1);
7907 let tmp2 = _mm_unpacklo_epi32(v2, v3);
7908 let tmp3 = _mm_unpackhi_epi32(v2, v3);
7909 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
7910 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
7911 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
7912 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
7913 let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
7914 let out0 = _mm_shuffle_epi8(out0, mask);
7915 let out1 = _mm_shuffle_epi8(out1, mask);
7916 let out2 = _mm_shuffle_epi8(out2, mask);
7917 let out3 = _mm_shuffle_epi8(out3, mask);
7918 let (chunks, []) = dest.as_chunks_mut::<16usize>() else {
7919 unreachable!()
7920 };
7921 crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
7922 out0,
7923 &mut chunks[0],
7924 );
7925 crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
7926 out1,
7927 &mut chunks[1],
7928 );
7929 crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
7930 out2,
7931 &mut chunks[2],
7932 );
7933 crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
7934 out3,
7935 &mut chunks[3],
7936 );
7937 }
7938 );
7939 kernel(self, a, dest);
7940 }
7941 #[inline(always)]
7942 fn reinterpret_u32_u8x64(self, a: u8x64<Self>) -> u32x16<Self> {
7943 let (a0, a1) = self.split_u8x64(a);
7944 self.combine_u32x8(
7945 self.reinterpret_u32_u8x32(a0),
7946 self.reinterpret_u32_u8x32(a1),
7947 )
7948 }
7949 #[inline(always)]
7950 fn splat_mask8x64(self, val: bool) -> mask8x64<Self> {
7951 let half = self.splat_mask8x32(val);
7952 self.combine_mask8x32(half, half)
7953 }
7954 #[inline(always)]
7955 fn load_array_mask8x64(self, val: [i8; 64usize]) -> mask8x64<Self> {
7956 mask8x64 {
7957 val: crate::transmute::checked_transmute_copy(&val),
7958 simd: self,
7959 }
7960 }
7961 #[inline(always)]
7962 fn as_array_mask8x64(self, a: mask8x64<Self>) -> [i8; 64usize] {
7963 crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i8; 64usize]>(&a.val.0)
7964 }
7965 #[inline(always)]
7966 fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64<Self> {
7967 crate::kernel!(
7968 #[inline(always)]
7969 fn kernel(token: Sse4_2, bits: u64) -> mask8x64<Sse4_2> {
7970 {
7971 let bit_bytes = _mm_set1_epi64x(bits.cast_signed());
7972 let bit_mask =
7973 _mm_setr_epi8(1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128);
7974 mask8x64 {
7975 val: crate::support::Aligned512([
7976 {
7977 let bit_bytes = _mm_shuffle_epi8(
7978 bit_bytes,
7979 _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1),
7980 );
7981 _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask)
7982 },
7983 {
7984 let bit_bytes = _mm_shuffle_epi8(
7985 bit_bytes,
7986 _mm_setr_epi8(2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3),
7987 );
7988 _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask)
7989 },
7990 {
7991 let bit_bytes = _mm_shuffle_epi8(
7992 bit_bytes,
7993 _mm_setr_epi8(4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5),
7994 );
7995 _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask)
7996 },
7997 {
7998 let bit_bytes = _mm_shuffle_epi8(
7999 bit_bytes,
8000 _mm_setr_epi8(6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7),
8001 );
8002 _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask)
8003 },
8004 ]),
8005 simd: token,
8006 }
8007 }
8008 }
8009 );
8010 kernel(self, bits)
8011 }
8012 #[inline(always)]
8013 fn to_bitmask_mask8x64(self, a: mask8x64<Self>) -> u64 {
8014 let (lo, hi) = self.split_mask8x64(a);
8015 let lo = self.to_bitmask_mask8x32(lo);
8016 let hi = self.to_bitmask_mask8x32(hi);
8017 lo | (hi << 32usize)
8018 }
8019 #[inline(always)]
8020 fn set_mask8x64(self, a: &mut mask8x64<Self>, index: usize, value: bool) -> () {
8021 assert!(
8022 index < 64usize,
8023 "mask lane index {index} is out of bounds for {} lanes",
8024 64usize
8025 );
8026 let mut lanes = self.as_array_mask8x64(*a);
8027 lanes[index] = if value { !0 } else { 0 };
8028 *a = self.load_array_mask8x64(lanes);
8029 }
8030 #[inline(always)]
8031 fn and_mask8x64(self, a: mask8x64<Self>, b: mask8x64<Self>) -> mask8x64<Self> {
8032 let (a0, a1) = self.split_mask8x64(a);
8033 let (b0, b1) = self.split_mask8x64(b);
8034 self.combine_mask8x32(self.and_mask8x32(a0, b0), self.and_mask8x32(a1, b1))
8035 }
8036 #[inline(always)]
8037 fn or_mask8x64(self, a: mask8x64<Self>, b: mask8x64<Self>) -> mask8x64<Self> {
8038 let (a0, a1) = self.split_mask8x64(a);
8039 let (b0, b1) = self.split_mask8x64(b);
8040 self.combine_mask8x32(self.or_mask8x32(a0, b0), self.or_mask8x32(a1, b1))
8041 }
8042 #[inline(always)]
8043 fn xor_mask8x64(self, a: mask8x64<Self>, b: mask8x64<Self>) -> mask8x64<Self> {
8044 let (a0, a1) = self.split_mask8x64(a);
8045 let (b0, b1) = self.split_mask8x64(b);
8046 self.combine_mask8x32(self.xor_mask8x32(a0, b0), self.xor_mask8x32(a1, b1))
8047 }
8048 #[inline(always)]
8049 fn not_mask8x64(self, a: mask8x64<Self>) -> mask8x64<Self> {
8050 let (a0, a1) = self.split_mask8x64(a);
8051 self.combine_mask8x32(self.not_mask8x32(a0), self.not_mask8x32(a1))
8052 }
8053 #[inline(always)]
8054 fn select_mask8x64(
8055 self,
8056 a: mask8x64<Self>,
8057 b: mask8x64<Self>,
8058 c: mask8x64<Self>,
8059 ) -> mask8x64<Self> {
8060 let (a0, a1) = self.split_mask8x64(a);
8061 let (b0, b1) = self.split_mask8x64(b);
8062 let (c0, c1) = self.split_mask8x64(c);
8063 self.combine_mask8x32(
8064 self.select_mask8x32(a0, b0, c0),
8065 self.select_mask8x32(a1, b1, c1),
8066 )
8067 }
8068 #[inline(always)]
8069 fn simd_eq_mask8x64(self, a: mask8x64<Self>, b: mask8x64<Self>) -> mask8x64<Self> {
8070 let (a0, a1) = self.split_mask8x64(a);
8071 let (b0, b1) = self.split_mask8x64(b);
8072 self.combine_mask8x32(self.simd_eq_mask8x32(a0, b0), self.simd_eq_mask8x32(a1, b1))
8073 }
8074 #[inline(always)]
8075 fn any_true_mask8x64(self, a: mask8x64<Self>) -> bool {
8076 let (a0, a1) = self.split_mask8x64(a);
8077 self.any_true_mask8x32(a0) || self.any_true_mask8x32(a1)
8078 }
8079 #[inline(always)]
8080 fn all_true_mask8x64(self, a: mask8x64<Self>) -> bool {
8081 let (a0, a1) = self.split_mask8x64(a);
8082 self.all_true_mask8x32(a0) && self.all_true_mask8x32(a1)
8083 }
8084 #[inline(always)]
8085 fn any_false_mask8x64(self, a: mask8x64<Self>) -> bool {
8086 let (a0, a1) = self.split_mask8x64(a);
8087 self.any_false_mask8x32(a0) || self.any_false_mask8x32(a1)
8088 }
8089 #[inline(always)]
8090 fn all_false_mask8x64(self, a: mask8x64<Self>) -> bool {
8091 let (a0, a1) = self.split_mask8x64(a);
8092 self.all_false_mask8x32(a0) && self.all_false_mask8x32(a1)
8093 }
8094 #[inline(always)]
8095 fn split_mask8x64(self, a: mask8x64<Self>) -> (mask8x32<Self>, mask8x32<Self>) {
8096 (
8097 mask8x32 {
8098 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
8099 simd: self,
8100 },
8101 mask8x32 {
8102 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
8103 simd: self,
8104 },
8105 )
8106 }
8107 #[inline(always)]
8108 fn splat_i16x32(self, val: i16) -> i16x32<Self> {
8109 let half = self.splat_i16x16(val);
8110 self.combine_i16x16(half, half)
8111 }
8112 #[inline(always)]
8113 fn load_array_i16x32(self, val: [i16; 32usize]) -> i16x32<Self> {
8114 i16x32 {
8115 val: crate::transmute::checked_transmute_copy(&val),
8116 simd: self,
8117 }
8118 }
8119 #[inline(always)]
8120 fn load_array_ref_i16x32(self, val: &[i16; 32usize]) -> i16x32<Self> {
8121 i16x32 {
8122 val: crate::transmute::checked_transmute_copy(val),
8123 simd: self,
8124 }
8125 }
8126 #[inline(always)]
8127 fn as_array_i16x32(self, a: i16x32<Self>) -> [i16; 32usize] {
8128 crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i16; 32usize]>(&a.val.0)
8129 }
8130 #[inline(always)]
8131 fn as_array_ref_i16x32(self, a: &i16x32<Self>) -> &[i16; 32usize] {
8132 crate::transmute::checked_cast_ref::<[__m128i; 4usize], [i16; 32usize]>(&a.val.0)
8133 }
8134 #[inline(always)]
8135 fn as_array_mut_i16x32(self, a: &mut i16x32<Self>) -> &mut [i16; 32usize] {
8136 crate::transmute::checked_cast_mut::<[__m128i; 4usize], [i16; 32usize]>(&mut a.val.0)
8137 }
8138 #[inline(always)]
8139 fn store_array_i16x32(self, a: i16x32<Self>, dest: &mut [i16; 32usize]) -> () {
8140 crate::transmute::checked_transmute_store(a.val.0, dest);
8141 }
8142 #[inline(always)]
8143 fn cvt_from_bytes_i16x32(self, a: u8x64<Self>) -> i16x32<Self> {
8144 i16x32 {
8145 val: crate::transmute::checked_transmute_copy(&a.val),
8146 simd: self,
8147 }
8148 }
8149 #[inline(always)]
8150 fn cvt_to_bytes_i16x32(self, a: i16x32<Self>) -> u8x64<Self> {
8151 u8x64 {
8152 val: crate::transmute::checked_transmute_copy(&a.val),
8153 simd: self,
8154 }
8155 }
8156 #[inline(always)]
8157 fn slide_i16x32<const SHIFT: usize>(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8158 if SHIFT >= 32usize {
8159 return b;
8160 }
8161 let result = cross_block_alignr_128x4(
8162 self,
8163 self.cvt_to_bytes_i16x32(b).val.0,
8164 self.cvt_to_bytes_i16x32(a).val.0,
8165 SHIFT * 2usize,
8166 );
8167 self.cvt_from_bytes_i16x32(u8x64 {
8168 val: crate::support::Aligned512(result),
8169 simd: self,
8170 })
8171 }
8172 #[inline(always)]
8173 fn slide_within_blocks_i16x32<const SHIFT: usize>(
8174 self,
8175 a: i16x32<Self>,
8176 b: i16x32<Self>,
8177 ) -> i16x32<Self> {
8178 let (a0, a1) = self.split_i16x32(a);
8179 let (b0, b1) = self.split_i16x32(b);
8180 self.combine_i16x16(
8181 self.slide_within_blocks_i16x16::<SHIFT>(a0, b0),
8182 self.slide_within_blocks_i16x16::<SHIFT>(a1, b1),
8183 )
8184 }
8185 #[inline(always)]
8186 fn swizzle_dyn_within_blocks_i16x32(
8187 self,
8188 a: i16x32<Self>,
8189 indices: u8x64<Self>,
8190 ) -> i16x32<Self> {
8191 let (a0, a1) = self.split_i16x32(a);
8192 let (indices0, indices1) = self.split_u8x64(indices);
8193 self.combine_i16x16(
8194 self.swizzle_dyn_within_blocks_i16x16(a0, indices0),
8195 self.swizzle_dyn_within_blocks_i16x16(a1, indices1),
8196 )
8197 }
8198 #[inline(always)]
8199 fn add_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8200 let (a0, a1) = self.split_i16x32(a);
8201 let (b0, b1) = self.split_i16x32(b);
8202 self.combine_i16x16(self.add_i16x16(a0, b0), self.add_i16x16(a1, b1))
8203 }
8204 #[inline(always)]
8205 fn sub_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8206 let (a0, a1) = self.split_i16x32(a);
8207 let (b0, b1) = self.split_i16x32(b);
8208 self.combine_i16x16(self.sub_i16x16(a0, b0), self.sub_i16x16(a1, b1))
8209 }
8210 #[inline(always)]
8211 fn mul_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8212 let (a0, a1) = self.split_i16x32(a);
8213 let (b0, b1) = self.split_i16x32(b);
8214 self.combine_i16x16(self.mul_i16x16(a0, b0), self.mul_i16x16(a1, b1))
8215 }
8216 #[inline(always)]
8217 fn and_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8218 let (a0, a1) = self.split_i16x32(a);
8219 let (b0, b1) = self.split_i16x32(b);
8220 self.combine_i16x16(self.and_i16x16(a0, b0), self.and_i16x16(a1, b1))
8221 }
8222 #[inline(always)]
8223 fn or_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8224 let (a0, a1) = self.split_i16x32(a);
8225 let (b0, b1) = self.split_i16x32(b);
8226 self.combine_i16x16(self.or_i16x16(a0, b0), self.or_i16x16(a1, b1))
8227 }
8228 #[inline(always)]
8229 fn xor_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8230 let (a0, a1) = self.split_i16x32(a);
8231 let (b0, b1) = self.split_i16x32(b);
8232 self.combine_i16x16(self.xor_i16x16(a0, b0), self.xor_i16x16(a1, b1))
8233 }
8234 #[inline(always)]
8235 fn not_i16x32(self, a: i16x32<Self>) -> i16x32<Self> {
8236 let (a0, a1) = self.split_i16x32(a);
8237 self.combine_i16x16(self.not_i16x16(a0), self.not_i16x16(a1))
8238 }
8239 #[inline(always)]
8240 fn shl_i16x32(self, a: i16x32<Self>, shift: u32) -> i16x32<Self> {
8241 let (a0, a1) = self.split_i16x32(a);
8242 self.combine_i16x16(self.shl_i16x16(a0, shift), self.shl_i16x16(a1, shift))
8243 }
8244 #[inline(always)]
8245 fn shlv_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8246 let (a0, a1) = self.split_i16x32(a);
8247 let (b0, b1) = self.split_i16x32(b);
8248 self.combine_i16x16(self.shlv_i16x16(a0, b0), self.shlv_i16x16(a1, b1))
8249 }
8250 #[inline(always)]
8251 fn shr_i16x32(self, a: i16x32<Self>, shift: u32) -> i16x32<Self> {
8252 let (a0, a1) = self.split_i16x32(a);
8253 self.combine_i16x16(self.shr_i16x16(a0, shift), self.shr_i16x16(a1, shift))
8254 }
8255 #[inline(always)]
8256 fn shrv_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8257 let (a0, a1) = self.split_i16x32(a);
8258 let (b0, b1) = self.split_i16x32(b);
8259 self.combine_i16x16(self.shrv_i16x16(a0, b0), self.shrv_i16x16(a1, b1))
8260 }
8261 #[inline(always)]
8262 fn simd_eq_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
8263 let (a0, a1) = self.split_i16x32(a);
8264 let (b0, b1) = self.split_i16x32(b);
8265 self.combine_mask16x16(self.simd_eq_i16x16(a0, b0), self.simd_eq_i16x16(a1, b1))
8266 }
8267 #[inline(always)]
8268 fn simd_lt_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
8269 let (a0, a1) = self.split_i16x32(a);
8270 let (b0, b1) = self.split_i16x32(b);
8271 self.combine_mask16x16(self.simd_lt_i16x16(a0, b0), self.simd_lt_i16x16(a1, b1))
8272 }
8273 #[inline(always)]
8274 fn simd_le_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
8275 let (a0, a1) = self.split_i16x32(a);
8276 let (b0, b1) = self.split_i16x32(b);
8277 self.combine_mask16x16(self.simd_le_i16x16(a0, b0), self.simd_le_i16x16(a1, b1))
8278 }
8279 #[inline(always)]
8280 fn simd_ge_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
8281 let (a0, a1) = self.split_i16x32(a);
8282 let (b0, b1) = self.split_i16x32(b);
8283 self.combine_mask16x16(self.simd_ge_i16x16(a0, b0), self.simd_ge_i16x16(a1, b1))
8284 }
8285 #[inline(always)]
8286 fn simd_gt_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
8287 let (a0, a1) = self.split_i16x32(a);
8288 let (b0, b1) = self.split_i16x32(b);
8289 self.combine_mask16x16(self.simd_gt_i16x16(a0, b0), self.simd_gt_i16x16(a1, b1))
8290 }
8291 #[inline(always)]
8292 fn zip_low_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8293 let (a0, _) = self.split_i16x32(a);
8294 let (b0, _) = self.split_i16x32(b);
8295 self.combine_i16x16(self.zip_low_i16x16(a0, b0), self.zip_high_i16x16(a0, b0))
8296 }
8297 #[inline(always)]
8298 fn zip_high_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8299 let (_, a1) = self.split_i16x32(a);
8300 let (_, b1) = self.split_i16x32(b);
8301 self.combine_i16x16(self.zip_low_i16x16(a1, b1), self.zip_high_i16x16(a1, b1))
8302 }
8303 #[inline(always)]
8304 fn unzip_low_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8305 let (a0, a1) = self.split_i16x32(a);
8306 let (b0, b1) = self.split_i16x32(b);
8307 self.combine_i16x16(self.unzip_low_i16x16(a0, a1), self.unzip_low_i16x16(b0, b1))
8308 }
8309 #[inline(always)]
8310 fn unzip_high_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8311 let (a0, a1) = self.split_i16x32(a);
8312 let (b0, b1) = self.split_i16x32(b);
8313 self.combine_i16x16(
8314 self.unzip_high_i16x16(a0, a1),
8315 self.unzip_high_i16x16(b0, b1),
8316 )
8317 }
8318 #[inline(always)]
8319 fn interleave_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> (i16x32<Self>, i16x32<Self>) {
8320 let (a0, a1) = self.split_i16x32(a);
8321 let (b0, b1) = self.split_i16x32(b);
8322 let lo_lo = self.zip_low_i16x16(a0, b0);
8323 let lo_hi = self.zip_high_i16x16(a0, b0);
8324 let hi_lo = self.zip_low_i16x16(a1, b1);
8325 let hi_hi = self.zip_high_i16x16(a1, b1);
8326 (
8327 self.combine_i16x16(lo_lo, lo_hi),
8328 self.combine_i16x16(hi_lo, hi_hi),
8329 )
8330 }
8331 #[inline(always)]
8332 fn deinterleave_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> (i16x32<Self>, i16x32<Self>) {
8333 let (a0, a1) = self.split_i16x32(a);
8334 let (b0, b1) = self.split_i16x32(b);
8335 let lo_even = self.unzip_low_i16x16(a0, a1);
8336 let lo_odd = self.unzip_high_i16x16(a0, a1);
8337 let hi_even = self.unzip_low_i16x16(b0, b1);
8338 let hi_odd = self.unzip_high_i16x16(b0, b1);
8339 (
8340 self.combine_i16x16(lo_even, hi_even),
8341 self.combine_i16x16(lo_odd, hi_odd),
8342 )
8343 }
8344 #[inline(always)]
8345 fn select_i16x32(self, a: mask16x32<Self>, b: i16x32<Self>, c: i16x32<Self>) -> i16x32<Self> {
8346 let (a0, a1) = self.split_mask16x32(a);
8347 let (b0, b1) = self.split_i16x32(b);
8348 let (c0, c1) = self.split_i16x32(c);
8349 self.combine_i16x16(
8350 self.select_i16x16(a0, b0, c0),
8351 self.select_i16x16(a1, b1, c1),
8352 )
8353 }
8354 #[inline(always)]
8355 fn min_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8356 let (a0, a1) = self.split_i16x32(a);
8357 let (b0, b1) = self.split_i16x32(b);
8358 self.combine_i16x16(self.min_i16x16(a0, b0), self.min_i16x16(a1, b1))
8359 }
8360 #[inline(always)]
8361 fn max_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8362 let (a0, a1) = self.split_i16x32(a);
8363 let (b0, b1) = self.split_i16x32(b);
8364 self.combine_i16x16(self.max_i16x16(a0, b0), self.max_i16x16(a1, b1))
8365 }
8366 #[inline(always)]
8367 fn split_i16x32(self, a: i16x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
8368 (
8369 i16x16 {
8370 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
8371 simd: self,
8372 },
8373 i16x16 {
8374 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
8375 simd: self,
8376 },
8377 )
8378 }
8379 #[inline(always)]
8380 fn neg_i16x32(self, a: i16x32<Self>) -> i16x32<Self> {
8381 let (a0, a1) = self.split_i16x32(a);
8382 self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1))
8383 }
8384 #[inline(always)]
8385 fn reinterpret_u8_i16x32(self, a: i16x32<Self>) -> u8x64<Self> {
8386 let (a0, a1) = self.split_i16x32(a);
8387 self.combine_u8x32(
8388 self.reinterpret_u8_i16x16(a0),
8389 self.reinterpret_u8_i16x16(a1),
8390 )
8391 }
8392 #[inline(always)]
8393 fn reinterpret_u32_i16x32(self, a: i16x32<Self>) -> u32x16<Self> {
8394 let (a0, a1) = self.split_i16x32(a);
8395 self.combine_u32x8(
8396 self.reinterpret_u32_i16x16(a0),
8397 self.reinterpret_u32_i16x16(a1),
8398 )
8399 }
8400 #[inline(always)]
8401 fn splat_u16x32(self, val: u16) -> u16x32<Self> {
8402 let half = self.splat_u16x16(val);
8403 self.combine_u16x16(half, half)
8404 }
8405 #[inline(always)]
8406 fn load_array_u16x32(self, val: [u16; 32usize]) -> u16x32<Self> {
8407 u16x32 {
8408 val: crate::transmute::checked_transmute_copy(&val),
8409 simd: self,
8410 }
8411 }
8412 #[inline(always)]
8413 fn load_array_ref_u16x32(self, val: &[u16; 32usize]) -> u16x32<Self> {
8414 u16x32 {
8415 val: crate::transmute::checked_transmute_copy(val),
8416 simd: self,
8417 }
8418 }
8419 #[inline(always)]
8420 fn as_array_u16x32(self, a: u16x32<Self>) -> [u16; 32usize] {
8421 crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [u16; 32usize]>(&a.val.0)
8422 }
8423 #[inline(always)]
8424 fn as_array_ref_u16x32(self, a: &u16x32<Self>) -> &[u16; 32usize] {
8425 crate::transmute::checked_cast_ref::<[__m128i; 4usize], [u16; 32usize]>(&a.val.0)
8426 }
8427 #[inline(always)]
8428 fn as_array_mut_u16x32(self, a: &mut u16x32<Self>) -> &mut [u16; 32usize] {
8429 crate::transmute::checked_cast_mut::<[__m128i; 4usize], [u16; 32usize]>(&mut a.val.0)
8430 }
8431 #[inline(always)]
8432 fn store_array_u16x32(self, a: u16x32<Self>, dest: &mut [u16; 32usize]) -> () {
8433 crate::transmute::checked_transmute_store(a.val.0, dest);
8434 }
8435 #[inline(always)]
8436 fn cvt_from_bytes_u16x32(self, a: u8x64<Self>) -> u16x32<Self> {
8437 u16x32 {
8438 val: crate::transmute::checked_transmute_copy(&a.val),
8439 simd: self,
8440 }
8441 }
8442 #[inline(always)]
8443 fn cvt_to_bytes_u16x32(self, a: u16x32<Self>) -> u8x64<Self> {
8444 u8x64 {
8445 val: crate::transmute::checked_transmute_copy(&a.val),
8446 simd: self,
8447 }
8448 }
8449 #[inline(always)]
8450 fn slide_u16x32<const SHIFT: usize>(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8451 if SHIFT >= 32usize {
8452 return b;
8453 }
8454 let result = cross_block_alignr_128x4(
8455 self,
8456 self.cvt_to_bytes_u16x32(b).val.0,
8457 self.cvt_to_bytes_u16x32(a).val.0,
8458 SHIFT * 2usize,
8459 );
8460 self.cvt_from_bytes_u16x32(u8x64 {
8461 val: crate::support::Aligned512(result),
8462 simd: self,
8463 })
8464 }
8465 #[inline(always)]
8466 fn slide_within_blocks_u16x32<const SHIFT: usize>(
8467 self,
8468 a: u16x32<Self>,
8469 b: u16x32<Self>,
8470 ) -> u16x32<Self> {
8471 let (a0, a1) = self.split_u16x32(a);
8472 let (b0, b1) = self.split_u16x32(b);
8473 self.combine_u16x16(
8474 self.slide_within_blocks_u16x16::<SHIFT>(a0, b0),
8475 self.slide_within_blocks_u16x16::<SHIFT>(a1, b1),
8476 )
8477 }
8478 #[inline(always)]
8479 fn swizzle_dyn_within_blocks_u16x32(
8480 self,
8481 a: u16x32<Self>,
8482 indices: u8x64<Self>,
8483 ) -> u16x32<Self> {
8484 let (a0, a1) = self.split_u16x32(a);
8485 let (indices0, indices1) = self.split_u8x64(indices);
8486 self.combine_u16x16(
8487 self.swizzle_dyn_within_blocks_u16x16(a0, indices0),
8488 self.swizzle_dyn_within_blocks_u16x16(a1, indices1),
8489 )
8490 }
8491 #[inline(always)]
8492 fn add_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8493 let (a0, a1) = self.split_u16x32(a);
8494 let (b0, b1) = self.split_u16x32(b);
8495 self.combine_u16x16(self.add_u16x16(a0, b0), self.add_u16x16(a1, b1))
8496 }
8497 #[inline(always)]
8498 fn sub_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8499 let (a0, a1) = self.split_u16x32(a);
8500 let (b0, b1) = self.split_u16x32(b);
8501 self.combine_u16x16(self.sub_u16x16(a0, b0), self.sub_u16x16(a1, b1))
8502 }
8503 #[inline(always)]
8504 fn mul_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8505 let (a0, a1) = self.split_u16x32(a);
8506 let (b0, b1) = self.split_u16x32(b);
8507 self.combine_u16x16(self.mul_u16x16(a0, b0), self.mul_u16x16(a1, b1))
8508 }
8509 #[inline(always)]
8510 fn and_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8511 let (a0, a1) = self.split_u16x32(a);
8512 let (b0, b1) = self.split_u16x32(b);
8513 self.combine_u16x16(self.and_u16x16(a0, b0), self.and_u16x16(a1, b1))
8514 }
8515 #[inline(always)]
8516 fn or_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8517 let (a0, a1) = self.split_u16x32(a);
8518 let (b0, b1) = self.split_u16x32(b);
8519 self.combine_u16x16(self.or_u16x16(a0, b0), self.or_u16x16(a1, b1))
8520 }
8521 #[inline(always)]
8522 fn xor_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8523 let (a0, a1) = self.split_u16x32(a);
8524 let (b0, b1) = self.split_u16x32(b);
8525 self.combine_u16x16(self.xor_u16x16(a0, b0), self.xor_u16x16(a1, b1))
8526 }
8527 #[inline(always)]
8528 fn not_u16x32(self, a: u16x32<Self>) -> u16x32<Self> {
8529 let (a0, a1) = self.split_u16x32(a);
8530 self.combine_u16x16(self.not_u16x16(a0), self.not_u16x16(a1))
8531 }
8532 #[inline(always)]
8533 fn shl_u16x32(self, a: u16x32<Self>, shift: u32) -> u16x32<Self> {
8534 let (a0, a1) = self.split_u16x32(a);
8535 self.combine_u16x16(self.shl_u16x16(a0, shift), self.shl_u16x16(a1, shift))
8536 }
8537 #[inline(always)]
8538 fn shlv_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8539 let (a0, a1) = self.split_u16x32(a);
8540 let (b0, b1) = self.split_u16x32(b);
8541 self.combine_u16x16(self.shlv_u16x16(a0, b0), self.shlv_u16x16(a1, b1))
8542 }
8543 #[inline(always)]
8544 fn shr_u16x32(self, a: u16x32<Self>, shift: u32) -> u16x32<Self> {
8545 let (a0, a1) = self.split_u16x32(a);
8546 self.combine_u16x16(self.shr_u16x16(a0, shift), self.shr_u16x16(a1, shift))
8547 }
8548 #[inline(always)]
8549 fn shrv_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8550 let (a0, a1) = self.split_u16x32(a);
8551 let (b0, b1) = self.split_u16x32(b);
8552 self.combine_u16x16(self.shrv_u16x16(a0, b0), self.shrv_u16x16(a1, b1))
8553 }
8554 #[inline(always)]
8555 fn simd_eq_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
8556 let (a0, a1) = self.split_u16x32(a);
8557 let (b0, b1) = self.split_u16x32(b);
8558 self.combine_mask16x16(self.simd_eq_u16x16(a0, b0), self.simd_eq_u16x16(a1, b1))
8559 }
8560 #[inline(always)]
8561 fn simd_lt_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
8562 let (a0, a1) = self.split_u16x32(a);
8563 let (b0, b1) = self.split_u16x32(b);
8564 self.combine_mask16x16(self.simd_lt_u16x16(a0, b0), self.simd_lt_u16x16(a1, b1))
8565 }
8566 #[inline(always)]
8567 fn simd_le_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
8568 let (a0, a1) = self.split_u16x32(a);
8569 let (b0, b1) = self.split_u16x32(b);
8570 self.combine_mask16x16(self.simd_le_u16x16(a0, b0), self.simd_le_u16x16(a1, b1))
8571 }
8572 #[inline(always)]
8573 fn simd_ge_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
8574 let (a0, a1) = self.split_u16x32(a);
8575 let (b0, b1) = self.split_u16x32(b);
8576 self.combine_mask16x16(self.simd_ge_u16x16(a0, b0), self.simd_ge_u16x16(a1, b1))
8577 }
8578 #[inline(always)]
8579 fn simd_gt_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
8580 let (a0, a1) = self.split_u16x32(a);
8581 let (b0, b1) = self.split_u16x32(b);
8582 self.combine_mask16x16(self.simd_gt_u16x16(a0, b0), self.simd_gt_u16x16(a1, b1))
8583 }
8584 #[inline(always)]
8585 fn zip_low_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8586 let (a0, _) = self.split_u16x32(a);
8587 let (b0, _) = self.split_u16x32(b);
8588 self.combine_u16x16(self.zip_low_u16x16(a0, b0), self.zip_high_u16x16(a0, b0))
8589 }
8590 #[inline(always)]
8591 fn zip_high_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8592 let (_, a1) = self.split_u16x32(a);
8593 let (_, b1) = self.split_u16x32(b);
8594 self.combine_u16x16(self.zip_low_u16x16(a1, b1), self.zip_high_u16x16(a1, b1))
8595 }
8596 #[inline(always)]
8597 fn unzip_low_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8598 let (a0, a1) = self.split_u16x32(a);
8599 let (b0, b1) = self.split_u16x32(b);
8600 self.combine_u16x16(self.unzip_low_u16x16(a0, a1), self.unzip_low_u16x16(b0, b1))
8601 }
8602 #[inline(always)]
8603 fn unzip_high_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8604 let (a0, a1) = self.split_u16x32(a);
8605 let (b0, b1) = self.split_u16x32(b);
8606 self.combine_u16x16(
8607 self.unzip_high_u16x16(a0, a1),
8608 self.unzip_high_u16x16(b0, b1),
8609 )
8610 }
8611 #[inline(always)]
8612 fn interleave_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> (u16x32<Self>, u16x32<Self>) {
8613 let (a0, a1) = self.split_u16x32(a);
8614 let (b0, b1) = self.split_u16x32(b);
8615 let lo_lo = self.zip_low_u16x16(a0, b0);
8616 let lo_hi = self.zip_high_u16x16(a0, b0);
8617 let hi_lo = self.zip_low_u16x16(a1, b1);
8618 let hi_hi = self.zip_high_u16x16(a1, b1);
8619 (
8620 self.combine_u16x16(lo_lo, lo_hi),
8621 self.combine_u16x16(hi_lo, hi_hi),
8622 )
8623 }
8624 #[inline(always)]
8625 fn deinterleave_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> (u16x32<Self>, u16x32<Self>) {
8626 let (a0, a1) = self.split_u16x32(a);
8627 let (b0, b1) = self.split_u16x32(b);
8628 let lo_even = self.unzip_low_u16x16(a0, a1);
8629 let lo_odd = self.unzip_high_u16x16(a0, a1);
8630 let hi_even = self.unzip_low_u16x16(b0, b1);
8631 let hi_odd = self.unzip_high_u16x16(b0, b1);
8632 (
8633 self.combine_u16x16(lo_even, hi_even),
8634 self.combine_u16x16(lo_odd, hi_odd),
8635 )
8636 }
8637 #[inline(always)]
8638 fn select_u16x32(self, a: mask16x32<Self>, b: u16x32<Self>, c: u16x32<Self>) -> u16x32<Self> {
8639 let (a0, a1) = self.split_mask16x32(a);
8640 let (b0, b1) = self.split_u16x32(b);
8641 let (c0, c1) = self.split_u16x32(c);
8642 self.combine_u16x16(
8643 self.select_u16x16(a0, b0, c0),
8644 self.select_u16x16(a1, b1, c1),
8645 )
8646 }
8647 #[inline(always)]
8648 fn min_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8649 let (a0, a1) = self.split_u16x32(a);
8650 let (b0, b1) = self.split_u16x32(b);
8651 self.combine_u16x16(self.min_u16x16(a0, b0), self.min_u16x16(a1, b1))
8652 }
8653 #[inline(always)]
8654 fn max_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
8655 let (a0, a1) = self.split_u16x32(a);
8656 let (b0, b1) = self.split_u16x32(b);
8657 self.combine_u16x16(self.max_u16x16(a0, b0), self.max_u16x16(a1, b1))
8658 }
8659 #[inline(always)]
8660 fn split_u16x32(self, a: u16x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
8661 (
8662 u16x16 {
8663 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
8664 simd: self,
8665 },
8666 u16x16 {
8667 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
8668 simd: self,
8669 },
8670 )
8671 }
8672 #[inline(always)]
8673 fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32<Self> {
8674 crate::kernel!(
8675 #[inline(always)]
8676 fn kernel(token: Sse4_2, src: &[u16; 32usize]) -> u16x32<Sse4_2> {
8677 let (chunks, []) = src.as_chunks::<8usize>() else {
8678 unreachable!()
8679 };
8680 let v0: __m128i =
8681 crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]);
8682 let v1: __m128i =
8683 crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]);
8684 let v2: __m128i =
8685 crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]);
8686 let v3: __m128i =
8687 crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]);
8688 let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
8689 let v0 = _mm_shuffle_epi8(v0, mask);
8690 let v1 = _mm_shuffle_epi8(v1, mask);
8691 let v2 = _mm_shuffle_epi8(v2, mask);
8692 let v3 = _mm_shuffle_epi8(v3, mask);
8693 let tmp0 = _mm_unpacklo_epi32(v0, v1);
8694 let tmp1 = _mm_unpackhi_epi32(v0, v1);
8695 let tmp2 = _mm_unpacklo_epi32(v2, v3);
8696 let tmp3 = _mm_unpackhi_epi32(v2, v3);
8697 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
8698 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
8699 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
8700 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
8701 token.combine_u16x16(
8702 token.combine_u16x8(out0.simd_into(token), out1.simd_into(token)),
8703 token.combine_u16x8(out2.simd_into(token), out3.simd_into(token)),
8704 )
8705 }
8706 );
8707 kernel(self, src)
8708 }
8709 #[inline(always)]
8710 fn store_interleaved_128_u16x32(self, a: u16x32<Self>, dest: &mut [u16; 32usize]) -> () {
8711 crate::kernel!(
8712 #[inline(always)]
8713 fn kernel(token: Sse4_2, a: u16x32<Sse4_2>, dest: &mut [u16; 32usize]) -> () {
8714 let (v01, v23) = token.split_u16x32(a);
8715 let (v0, v1) = token.split_u16x16(v01);
8716 let (v2, v3) = token.split_u16x16(v23);
8717 let v0 = v0.into();
8718 let v1 = v1.into();
8719 let v2 = v2.into();
8720 let v3 = v3.into();
8721 let tmp0 = _mm_unpacklo_epi32(v0, v1);
8722 let tmp1 = _mm_unpackhi_epi32(v0, v1);
8723 let tmp2 = _mm_unpacklo_epi32(v2, v3);
8724 let tmp3 = _mm_unpackhi_epi32(v2, v3);
8725 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
8726 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
8727 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
8728 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
8729 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
8730 let out0 = _mm_shuffle_epi8(out0, mask);
8731 let out1 = _mm_shuffle_epi8(out1, mask);
8732 let out2 = _mm_shuffle_epi8(out2, mask);
8733 let out3 = _mm_shuffle_epi8(out3, mask);
8734 let (chunks, []) = dest.as_chunks_mut::<8usize>() else {
8735 unreachable!()
8736 };
8737 crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
8738 out0,
8739 &mut chunks[0],
8740 );
8741 crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
8742 out1,
8743 &mut chunks[1],
8744 );
8745 crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
8746 out2,
8747 &mut chunks[2],
8748 );
8749 crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
8750 out3,
8751 &mut chunks[3],
8752 );
8753 }
8754 );
8755 kernel(self, a, dest);
8756 }
8757 #[inline(always)]
8758 fn narrow_u16x32(self, a: u16x32<Self>) -> u8x32<Self> {
8759 let (a0, a1) = self.split_u16x32(a);
8760 self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1))
8761 }
8762 #[inline(always)]
8763 fn reinterpret_u8_u16x32(self, a: u16x32<Self>) -> u8x64<Self> {
8764 let (a0, a1) = self.split_u16x32(a);
8765 self.combine_u8x32(
8766 self.reinterpret_u8_u16x16(a0),
8767 self.reinterpret_u8_u16x16(a1),
8768 )
8769 }
8770 #[inline(always)]
8771 fn reinterpret_u32_u16x32(self, a: u16x32<Self>) -> u32x16<Self> {
8772 let (a0, a1) = self.split_u16x32(a);
8773 self.combine_u32x8(
8774 self.reinterpret_u32_u16x16(a0),
8775 self.reinterpret_u32_u16x16(a1),
8776 )
8777 }
8778 #[inline(always)]
8779 fn splat_mask16x32(self, val: bool) -> mask16x32<Self> {
8780 let half = self.splat_mask16x16(val);
8781 self.combine_mask16x16(half, half)
8782 }
8783 #[inline(always)]
8784 fn load_array_mask16x32(self, val: [i16; 32usize]) -> mask16x32<Self> {
8785 mask16x32 {
8786 val: crate::transmute::checked_transmute_copy(&val),
8787 simd: self,
8788 }
8789 }
8790 #[inline(always)]
8791 fn as_array_mask16x32(self, a: mask16x32<Self>) -> [i16; 32usize] {
8792 crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i16; 32usize]>(&a.val.0)
8793 }
8794 #[inline(always)]
8795 fn from_bitmask_mask16x32(self, bits: u64) -> mask16x32<Self> {
8796 let lo = self.from_bitmask_mask16x16(bits);
8797 let hi = self.from_bitmask_mask16x16(bits >> 16usize);
8798 self.combine_mask16x16(lo, hi)
8799 }
8800 #[inline(always)]
8801 fn to_bitmask_mask16x32(self, a: mask16x32<Self>) -> u64 {
8802 crate::kernel!(
8803 #[inline(always)]
8804 fn kernel(token: Sse4_2, a: mask16x32<Sse4_2>) -> u64 {
8805 {
8806 let lo = _mm_packs_epi16(a.val.0[0], a.val.0[1]);
8807 let hi = _mm_packs_epi16(a.val.0[2], a.val.0[3]);
8808 let lo = _mm_movemask_epi8(lo) as u32 as u64;
8809 let hi = _mm_movemask_epi8(hi) as u32 as u64;
8810 lo | (hi << 16usize)
8811 }
8812 }
8813 );
8814 kernel(self, a)
8815 }
8816 #[inline(always)]
8817 fn set_mask16x32(self, a: &mut mask16x32<Self>, index: usize, value: bool) -> () {
8818 assert!(
8819 index < 32usize,
8820 "mask lane index {index} is out of bounds for {} lanes",
8821 32usize
8822 );
8823 let mut lanes = self.as_array_mask16x32(*a);
8824 lanes[index] = if value { !0 } else { 0 };
8825 *a = self.load_array_mask16x32(lanes);
8826 }
8827 #[inline(always)]
8828 fn and_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask16x32<Self> {
8829 let (a0, a1) = self.split_mask16x32(a);
8830 let (b0, b1) = self.split_mask16x32(b);
8831 self.combine_mask16x16(self.and_mask16x16(a0, b0), self.and_mask16x16(a1, b1))
8832 }
8833 #[inline(always)]
8834 fn or_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask16x32<Self> {
8835 let (a0, a1) = self.split_mask16x32(a);
8836 let (b0, b1) = self.split_mask16x32(b);
8837 self.combine_mask16x16(self.or_mask16x16(a0, b0), self.or_mask16x16(a1, b1))
8838 }
8839 #[inline(always)]
8840 fn xor_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask16x32<Self> {
8841 let (a0, a1) = self.split_mask16x32(a);
8842 let (b0, b1) = self.split_mask16x32(b);
8843 self.combine_mask16x16(self.xor_mask16x16(a0, b0), self.xor_mask16x16(a1, b1))
8844 }
8845 #[inline(always)]
8846 fn not_mask16x32(self, a: mask16x32<Self>) -> mask16x32<Self> {
8847 let (a0, a1) = self.split_mask16x32(a);
8848 self.combine_mask16x16(self.not_mask16x16(a0), self.not_mask16x16(a1))
8849 }
8850 #[inline(always)]
8851 fn select_mask16x32(
8852 self,
8853 a: mask16x32<Self>,
8854 b: mask16x32<Self>,
8855 c: mask16x32<Self>,
8856 ) -> mask16x32<Self> {
8857 let (a0, a1) = self.split_mask16x32(a);
8858 let (b0, b1) = self.split_mask16x32(b);
8859 let (c0, c1) = self.split_mask16x32(c);
8860 self.combine_mask16x16(
8861 self.select_mask16x16(a0, b0, c0),
8862 self.select_mask16x16(a1, b1, c1),
8863 )
8864 }
8865 #[inline(always)]
8866 fn simd_eq_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask16x32<Self> {
8867 let (a0, a1) = self.split_mask16x32(a);
8868 let (b0, b1) = self.split_mask16x32(b);
8869 self.combine_mask16x16(
8870 self.simd_eq_mask16x16(a0, b0),
8871 self.simd_eq_mask16x16(a1, b1),
8872 )
8873 }
8874 #[inline(always)]
8875 fn any_true_mask16x32(self, a: mask16x32<Self>) -> bool {
8876 let (a0, a1) = self.split_mask16x32(a);
8877 self.any_true_mask16x16(a0) || self.any_true_mask16x16(a1)
8878 }
8879 #[inline(always)]
8880 fn all_true_mask16x32(self, a: mask16x32<Self>) -> bool {
8881 let (a0, a1) = self.split_mask16x32(a);
8882 self.all_true_mask16x16(a0) && self.all_true_mask16x16(a1)
8883 }
8884 #[inline(always)]
8885 fn any_false_mask16x32(self, a: mask16x32<Self>) -> bool {
8886 let (a0, a1) = self.split_mask16x32(a);
8887 self.any_false_mask16x16(a0) || self.any_false_mask16x16(a1)
8888 }
8889 #[inline(always)]
8890 fn all_false_mask16x32(self, a: mask16x32<Self>) -> bool {
8891 let (a0, a1) = self.split_mask16x32(a);
8892 self.all_false_mask16x16(a0) && self.all_false_mask16x16(a1)
8893 }
8894 #[inline(always)]
8895 fn split_mask16x32(self, a: mask16x32<Self>) -> (mask16x16<Self>, mask16x16<Self>) {
8896 (
8897 mask16x16 {
8898 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
8899 simd: self,
8900 },
8901 mask16x16 {
8902 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
8903 simd: self,
8904 },
8905 )
8906 }
8907 #[inline(always)]
8908 fn splat_i32x16(self, val: i32) -> i32x16<Self> {
8909 let half = self.splat_i32x8(val);
8910 self.combine_i32x8(half, half)
8911 }
8912 #[inline(always)]
8913 fn load_array_i32x16(self, val: [i32; 16usize]) -> i32x16<Self> {
8914 i32x16 {
8915 val: crate::transmute::checked_transmute_copy(&val),
8916 simd: self,
8917 }
8918 }
8919 #[inline(always)]
8920 fn load_array_ref_i32x16(self, val: &[i32; 16usize]) -> i32x16<Self> {
8921 i32x16 {
8922 val: crate::transmute::checked_transmute_copy(val),
8923 simd: self,
8924 }
8925 }
8926 #[inline(always)]
8927 fn as_array_i32x16(self, a: i32x16<Self>) -> [i32; 16usize] {
8928 crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i32; 16usize]>(&a.val.0)
8929 }
8930 #[inline(always)]
8931 fn as_array_ref_i32x16(self, a: &i32x16<Self>) -> &[i32; 16usize] {
8932 crate::transmute::checked_cast_ref::<[__m128i; 4usize], [i32; 16usize]>(&a.val.0)
8933 }
8934 #[inline(always)]
8935 fn as_array_mut_i32x16(self, a: &mut i32x16<Self>) -> &mut [i32; 16usize] {
8936 crate::transmute::checked_cast_mut::<[__m128i; 4usize], [i32; 16usize]>(&mut a.val.0)
8937 }
8938 #[inline(always)]
8939 fn store_array_i32x16(self, a: i32x16<Self>, dest: &mut [i32; 16usize]) -> () {
8940 crate::transmute::checked_transmute_store(a.val.0, dest);
8941 }
8942 #[inline(always)]
8943 fn cvt_from_bytes_i32x16(self, a: u8x64<Self>) -> i32x16<Self> {
8944 i32x16 {
8945 val: crate::transmute::checked_transmute_copy(&a.val),
8946 simd: self,
8947 }
8948 }
8949 #[inline(always)]
8950 fn cvt_to_bytes_i32x16(self, a: i32x16<Self>) -> u8x64<Self> {
8951 u8x64 {
8952 val: crate::transmute::checked_transmute_copy(&a.val),
8953 simd: self,
8954 }
8955 }
8956 #[inline(always)]
8957 fn slide_i32x16<const SHIFT: usize>(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
8958 if SHIFT >= 16usize {
8959 return b;
8960 }
8961 let result = cross_block_alignr_128x4(
8962 self,
8963 self.cvt_to_bytes_i32x16(b).val.0,
8964 self.cvt_to_bytes_i32x16(a).val.0,
8965 SHIFT * 4usize,
8966 );
8967 self.cvt_from_bytes_i32x16(u8x64 {
8968 val: crate::support::Aligned512(result),
8969 simd: self,
8970 })
8971 }
8972 #[inline(always)]
8973 fn slide_within_blocks_i32x16<const SHIFT: usize>(
8974 self,
8975 a: i32x16<Self>,
8976 b: i32x16<Self>,
8977 ) -> i32x16<Self> {
8978 let (a0, a1) = self.split_i32x16(a);
8979 let (b0, b1) = self.split_i32x16(b);
8980 self.combine_i32x8(
8981 self.slide_within_blocks_i32x8::<SHIFT>(a0, b0),
8982 self.slide_within_blocks_i32x8::<SHIFT>(a1, b1),
8983 )
8984 }
8985 #[inline(always)]
8986 fn swizzle_dyn_within_blocks_i32x16(
8987 self,
8988 a: i32x16<Self>,
8989 indices: u8x64<Self>,
8990 ) -> i32x16<Self> {
8991 let (a0, a1) = self.split_i32x16(a);
8992 let (indices0, indices1) = self.split_u8x64(indices);
8993 self.combine_i32x8(
8994 self.swizzle_dyn_within_blocks_i32x8(a0, indices0),
8995 self.swizzle_dyn_within_blocks_i32x8(a1, indices1),
8996 )
8997 }
8998 #[inline(always)]
8999 fn add_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9000 let (a0, a1) = self.split_i32x16(a);
9001 let (b0, b1) = self.split_i32x16(b);
9002 self.combine_i32x8(self.add_i32x8(a0, b0), self.add_i32x8(a1, b1))
9003 }
9004 #[inline(always)]
9005 fn sub_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9006 let (a0, a1) = self.split_i32x16(a);
9007 let (b0, b1) = self.split_i32x16(b);
9008 self.combine_i32x8(self.sub_i32x8(a0, b0), self.sub_i32x8(a1, b1))
9009 }
9010 #[inline(always)]
9011 fn mul_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9012 let (a0, a1) = self.split_i32x16(a);
9013 let (b0, b1) = self.split_i32x16(b);
9014 self.combine_i32x8(self.mul_i32x8(a0, b0), self.mul_i32x8(a1, b1))
9015 }
9016 #[inline(always)]
9017 fn and_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9018 let (a0, a1) = self.split_i32x16(a);
9019 let (b0, b1) = self.split_i32x16(b);
9020 self.combine_i32x8(self.and_i32x8(a0, b0), self.and_i32x8(a1, b1))
9021 }
9022 #[inline(always)]
9023 fn or_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9024 let (a0, a1) = self.split_i32x16(a);
9025 let (b0, b1) = self.split_i32x16(b);
9026 self.combine_i32x8(self.or_i32x8(a0, b0), self.or_i32x8(a1, b1))
9027 }
9028 #[inline(always)]
9029 fn xor_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9030 let (a0, a1) = self.split_i32x16(a);
9031 let (b0, b1) = self.split_i32x16(b);
9032 self.combine_i32x8(self.xor_i32x8(a0, b0), self.xor_i32x8(a1, b1))
9033 }
9034 #[inline(always)]
9035 fn not_i32x16(self, a: i32x16<Self>) -> i32x16<Self> {
9036 let (a0, a1) = self.split_i32x16(a);
9037 self.combine_i32x8(self.not_i32x8(a0), self.not_i32x8(a1))
9038 }
9039 #[inline(always)]
9040 fn shl_i32x16(self, a: i32x16<Self>, shift: u32) -> i32x16<Self> {
9041 let (a0, a1) = self.split_i32x16(a);
9042 self.combine_i32x8(self.shl_i32x8(a0, shift), self.shl_i32x8(a1, shift))
9043 }
9044 #[inline(always)]
9045 fn shlv_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9046 let (a0, a1) = self.split_i32x16(a);
9047 let (b0, b1) = self.split_i32x16(b);
9048 self.combine_i32x8(self.shlv_i32x8(a0, b0), self.shlv_i32x8(a1, b1))
9049 }
9050 #[inline(always)]
9051 fn shr_i32x16(self, a: i32x16<Self>, shift: u32) -> i32x16<Self> {
9052 let (a0, a1) = self.split_i32x16(a);
9053 self.combine_i32x8(self.shr_i32x8(a0, shift), self.shr_i32x8(a1, shift))
9054 }
9055 #[inline(always)]
9056 fn shrv_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9057 let (a0, a1) = self.split_i32x16(a);
9058 let (b0, b1) = self.split_i32x16(b);
9059 self.combine_i32x8(self.shrv_i32x8(a0, b0), self.shrv_i32x8(a1, b1))
9060 }
9061 #[inline(always)]
9062 fn simd_eq_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
9063 let (a0, a1) = self.split_i32x16(a);
9064 let (b0, b1) = self.split_i32x16(b);
9065 self.combine_mask32x8(self.simd_eq_i32x8(a0, b0), self.simd_eq_i32x8(a1, b1))
9066 }
9067 #[inline(always)]
9068 fn simd_lt_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
9069 let (a0, a1) = self.split_i32x16(a);
9070 let (b0, b1) = self.split_i32x16(b);
9071 self.combine_mask32x8(self.simd_lt_i32x8(a0, b0), self.simd_lt_i32x8(a1, b1))
9072 }
9073 #[inline(always)]
9074 fn simd_le_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
9075 let (a0, a1) = self.split_i32x16(a);
9076 let (b0, b1) = self.split_i32x16(b);
9077 self.combine_mask32x8(self.simd_le_i32x8(a0, b0), self.simd_le_i32x8(a1, b1))
9078 }
9079 #[inline(always)]
9080 fn simd_ge_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
9081 let (a0, a1) = self.split_i32x16(a);
9082 let (b0, b1) = self.split_i32x16(b);
9083 self.combine_mask32x8(self.simd_ge_i32x8(a0, b0), self.simd_ge_i32x8(a1, b1))
9084 }
9085 #[inline(always)]
9086 fn simd_gt_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
9087 let (a0, a1) = self.split_i32x16(a);
9088 let (b0, b1) = self.split_i32x16(b);
9089 self.combine_mask32x8(self.simd_gt_i32x8(a0, b0), self.simd_gt_i32x8(a1, b1))
9090 }
9091 #[inline(always)]
9092 fn zip_low_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9093 let (a0, _) = self.split_i32x16(a);
9094 let (b0, _) = self.split_i32x16(b);
9095 self.combine_i32x8(self.zip_low_i32x8(a0, b0), self.zip_high_i32x8(a0, b0))
9096 }
9097 #[inline(always)]
9098 fn zip_high_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9099 let (_, a1) = self.split_i32x16(a);
9100 let (_, b1) = self.split_i32x16(b);
9101 self.combine_i32x8(self.zip_low_i32x8(a1, b1), self.zip_high_i32x8(a1, b1))
9102 }
9103 #[inline(always)]
9104 fn unzip_low_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9105 let (a0, a1) = self.split_i32x16(a);
9106 let (b0, b1) = self.split_i32x16(b);
9107 self.combine_i32x8(self.unzip_low_i32x8(a0, a1), self.unzip_low_i32x8(b0, b1))
9108 }
9109 #[inline(always)]
9110 fn unzip_high_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9111 let (a0, a1) = self.split_i32x16(a);
9112 let (b0, b1) = self.split_i32x16(b);
9113 self.combine_i32x8(self.unzip_high_i32x8(a0, a1), self.unzip_high_i32x8(b0, b1))
9114 }
9115 #[inline(always)]
9116 fn interleave_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> (i32x16<Self>, i32x16<Self>) {
9117 let (a0, a1) = self.split_i32x16(a);
9118 let (b0, b1) = self.split_i32x16(b);
9119 let lo_lo = self.zip_low_i32x8(a0, b0);
9120 let lo_hi = self.zip_high_i32x8(a0, b0);
9121 let hi_lo = self.zip_low_i32x8(a1, b1);
9122 let hi_hi = self.zip_high_i32x8(a1, b1);
9123 (
9124 self.combine_i32x8(lo_lo, lo_hi),
9125 self.combine_i32x8(hi_lo, hi_hi),
9126 )
9127 }
9128 #[inline(always)]
9129 fn deinterleave_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> (i32x16<Self>, i32x16<Self>) {
9130 let (a0, a1) = self.split_i32x16(a);
9131 let (b0, b1) = self.split_i32x16(b);
9132 let lo_even = self.unzip_low_i32x8(a0, a1);
9133 let lo_odd = self.unzip_high_i32x8(a0, a1);
9134 let hi_even = self.unzip_low_i32x8(b0, b1);
9135 let hi_odd = self.unzip_high_i32x8(b0, b1);
9136 (
9137 self.combine_i32x8(lo_even, hi_even),
9138 self.combine_i32x8(lo_odd, hi_odd),
9139 )
9140 }
9141 #[inline(always)]
9142 fn select_i32x16(self, a: mask32x16<Self>, b: i32x16<Self>, c: i32x16<Self>) -> i32x16<Self> {
9143 let (a0, a1) = self.split_mask32x16(a);
9144 let (b0, b1) = self.split_i32x16(b);
9145 let (c0, c1) = self.split_i32x16(c);
9146 self.combine_i32x8(self.select_i32x8(a0, b0, c0), self.select_i32x8(a1, b1, c1))
9147 }
9148 #[inline(always)]
9149 fn min_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9150 let (a0, a1) = self.split_i32x16(a);
9151 let (b0, b1) = self.split_i32x16(b);
9152 self.combine_i32x8(self.min_i32x8(a0, b0), self.min_i32x8(a1, b1))
9153 }
9154 #[inline(always)]
9155 fn max_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9156 let (a0, a1) = self.split_i32x16(a);
9157 let (b0, b1) = self.split_i32x16(b);
9158 self.combine_i32x8(self.max_i32x8(a0, b0), self.max_i32x8(a1, b1))
9159 }
9160 #[inline(always)]
9161 fn split_i32x16(self, a: i32x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
9162 (
9163 i32x8 {
9164 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9165 simd: self,
9166 },
9167 i32x8 {
9168 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9169 simd: self,
9170 },
9171 )
9172 }
9173 #[inline(always)]
9174 fn neg_i32x16(self, a: i32x16<Self>) -> i32x16<Self> {
9175 let (a0, a1) = self.split_i32x16(a);
9176 self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1))
9177 }
9178 #[inline(always)]
9179 fn reinterpret_u8_i32x16(self, a: i32x16<Self>) -> u8x64<Self> {
9180 let (a0, a1) = self.split_i32x16(a);
9181 self.combine_u8x32(self.reinterpret_u8_i32x8(a0), self.reinterpret_u8_i32x8(a1))
9182 }
9183 #[inline(always)]
9184 fn reinterpret_u32_i32x16(self, a: i32x16<Self>) -> u32x16<Self> {
9185 let (a0, a1) = self.split_i32x16(a);
9186 self.combine_u32x8(
9187 self.reinterpret_u32_i32x8(a0),
9188 self.reinterpret_u32_i32x8(a1),
9189 )
9190 }
9191 #[inline(always)]
9192 fn cvt_f32_i32x16(self, a: i32x16<Self>) -> f32x16<Self> {
9193 let (a0, a1) = self.split_i32x16(a);
9194 self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1))
9195 }
9196 #[inline(always)]
9197 fn splat_u32x16(self, val: u32) -> u32x16<Self> {
9198 let half = self.splat_u32x8(val);
9199 self.combine_u32x8(half, half)
9200 }
9201 #[inline(always)]
9202 fn load_array_u32x16(self, val: [u32; 16usize]) -> u32x16<Self> {
9203 u32x16 {
9204 val: crate::transmute::checked_transmute_copy(&val),
9205 simd: self,
9206 }
9207 }
9208 #[inline(always)]
9209 fn load_array_ref_u32x16(self, val: &[u32; 16usize]) -> u32x16<Self> {
9210 u32x16 {
9211 val: crate::transmute::checked_transmute_copy(val),
9212 simd: self,
9213 }
9214 }
9215 #[inline(always)]
9216 fn as_array_u32x16(self, a: u32x16<Self>) -> [u32; 16usize] {
9217 crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [u32; 16usize]>(&a.val.0)
9218 }
9219 #[inline(always)]
9220 fn as_array_ref_u32x16(self, a: &u32x16<Self>) -> &[u32; 16usize] {
9221 crate::transmute::checked_cast_ref::<[__m128i; 4usize], [u32; 16usize]>(&a.val.0)
9222 }
9223 #[inline(always)]
9224 fn as_array_mut_u32x16(self, a: &mut u32x16<Self>) -> &mut [u32; 16usize] {
9225 crate::transmute::checked_cast_mut::<[__m128i; 4usize], [u32; 16usize]>(&mut a.val.0)
9226 }
9227 #[inline(always)]
9228 fn store_array_u32x16(self, a: u32x16<Self>, dest: &mut [u32; 16usize]) -> () {
9229 crate::transmute::checked_transmute_store(a.val.0, dest);
9230 }
9231 #[inline(always)]
9232 fn cvt_from_bytes_u32x16(self, a: u8x64<Self>) -> u32x16<Self> {
9233 u32x16 {
9234 val: crate::transmute::checked_transmute_copy(&a.val),
9235 simd: self,
9236 }
9237 }
9238 #[inline(always)]
9239 fn cvt_to_bytes_u32x16(self, a: u32x16<Self>) -> u8x64<Self> {
9240 u8x64 {
9241 val: crate::transmute::checked_transmute_copy(&a.val),
9242 simd: self,
9243 }
9244 }
9245 #[inline(always)]
9246 fn slide_u32x16<const SHIFT: usize>(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9247 if SHIFT >= 16usize {
9248 return b;
9249 }
9250 let result = cross_block_alignr_128x4(
9251 self,
9252 self.cvt_to_bytes_u32x16(b).val.0,
9253 self.cvt_to_bytes_u32x16(a).val.0,
9254 SHIFT * 4usize,
9255 );
9256 self.cvt_from_bytes_u32x16(u8x64 {
9257 val: crate::support::Aligned512(result),
9258 simd: self,
9259 })
9260 }
9261 #[inline(always)]
9262 fn slide_within_blocks_u32x16<const SHIFT: usize>(
9263 self,
9264 a: u32x16<Self>,
9265 b: u32x16<Self>,
9266 ) -> u32x16<Self> {
9267 let (a0, a1) = self.split_u32x16(a);
9268 let (b0, b1) = self.split_u32x16(b);
9269 self.combine_u32x8(
9270 self.slide_within_blocks_u32x8::<SHIFT>(a0, b0),
9271 self.slide_within_blocks_u32x8::<SHIFT>(a1, b1),
9272 )
9273 }
9274 #[inline(always)]
9275 fn swizzle_dyn_within_blocks_u32x16(
9276 self,
9277 a: u32x16<Self>,
9278 indices: u8x64<Self>,
9279 ) -> u32x16<Self> {
9280 let (a0, a1) = self.split_u32x16(a);
9281 let (indices0, indices1) = self.split_u8x64(indices);
9282 self.combine_u32x8(
9283 self.swizzle_dyn_within_blocks_u32x8(a0, indices0),
9284 self.swizzle_dyn_within_blocks_u32x8(a1, indices1),
9285 )
9286 }
9287 #[inline(always)]
9288 fn add_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9289 let (a0, a1) = self.split_u32x16(a);
9290 let (b0, b1) = self.split_u32x16(b);
9291 self.combine_u32x8(self.add_u32x8(a0, b0), self.add_u32x8(a1, b1))
9292 }
9293 #[inline(always)]
9294 fn sub_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9295 let (a0, a1) = self.split_u32x16(a);
9296 let (b0, b1) = self.split_u32x16(b);
9297 self.combine_u32x8(self.sub_u32x8(a0, b0), self.sub_u32x8(a1, b1))
9298 }
9299 #[inline(always)]
9300 fn mul_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9301 let (a0, a1) = self.split_u32x16(a);
9302 let (b0, b1) = self.split_u32x16(b);
9303 self.combine_u32x8(self.mul_u32x8(a0, b0), self.mul_u32x8(a1, b1))
9304 }
9305 #[inline(always)]
9306 fn and_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9307 let (a0, a1) = self.split_u32x16(a);
9308 let (b0, b1) = self.split_u32x16(b);
9309 self.combine_u32x8(self.and_u32x8(a0, b0), self.and_u32x8(a1, b1))
9310 }
9311 #[inline(always)]
9312 fn or_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9313 let (a0, a1) = self.split_u32x16(a);
9314 let (b0, b1) = self.split_u32x16(b);
9315 self.combine_u32x8(self.or_u32x8(a0, b0), self.or_u32x8(a1, b1))
9316 }
9317 #[inline(always)]
9318 fn xor_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9319 let (a0, a1) = self.split_u32x16(a);
9320 let (b0, b1) = self.split_u32x16(b);
9321 self.combine_u32x8(self.xor_u32x8(a0, b0), self.xor_u32x8(a1, b1))
9322 }
9323 #[inline(always)]
9324 fn not_u32x16(self, a: u32x16<Self>) -> u32x16<Self> {
9325 let (a0, a1) = self.split_u32x16(a);
9326 self.combine_u32x8(self.not_u32x8(a0), self.not_u32x8(a1))
9327 }
9328 #[inline(always)]
9329 fn shl_u32x16(self, a: u32x16<Self>, shift: u32) -> u32x16<Self> {
9330 let (a0, a1) = self.split_u32x16(a);
9331 self.combine_u32x8(self.shl_u32x8(a0, shift), self.shl_u32x8(a1, shift))
9332 }
9333 #[inline(always)]
9334 fn shlv_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9335 let (a0, a1) = self.split_u32x16(a);
9336 let (b0, b1) = self.split_u32x16(b);
9337 self.combine_u32x8(self.shlv_u32x8(a0, b0), self.shlv_u32x8(a1, b1))
9338 }
9339 #[inline(always)]
9340 fn shr_u32x16(self, a: u32x16<Self>, shift: u32) -> u32x16<Self> {
9341 let (a0, a1) = self.split_u32x16(a);
9342 self.combine_u32x8(self.shr_u32x8(a0, shift), self.shr_u32x8(a1, shift))
9343 }
9344 #[inline(always)]
9345 fn shrv_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9346 let (a0, a1) = self.split_u32x16(a);
9347 let (b0, b1) = self.split_u32x16(b);
9348 self.combine_u32x8(self.shrv_u32x8(a0, b0), self.shrv_u32x8(a1, b1))
9349 }
9350 #[inline(always)]
9351 fn simd_eq_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
9352 let (a0, a1) = self.split_u32x16(a);
9353 let (b0, b1) = self.split_u32x16(b);
9354 self.combine_mask32x8(self.simd_eq_u32x8(a0, b0), self.simd_eq_u32x8(a1, b1))
9355 }
9356 #[inline(always)]
9357 fn simd_lt_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
9358 let (a0, a1) = self.split_u32x16(a);
9359 let (b0, b1) = self.split_u32x16(b);
9360 self.combine_mask32x8(self.simd_lt_u32x8(a0, b0), self.simd_lt_u32x8(a1, b1))
9361 }
9362 #[inline(always)]
9363 fn simd_le_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
9364 let (a0, a1) = self.split_u32x16(a);
9365 let (b0, b1) = self.split_u32x16(b);
9366 self.combine_mask32x8(self.simd_le_u32x8(a0, b0), self.simd_le_u32x8(a1, b1))
9367 }
9368 #[inline(always)]
9369 fn simd_ge_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
9370 let (a0, a1) = self.split_u32x16(a);
9371 let (b0, b1) = self.split_u32x16(b);
9372 self.combine_mask32x8(self.simd_ge_u32x8(a0, b0), self.simd_ge_u32x8(a1, b1))
9373 }
9374 #[inline(always)]
9375 fn simd_gt_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
9376 let (a0, a1) = self.split_u32x16(a);
9377 let (b0, b1) = self.split_u32x16(b);
9378 self.combine_mask32x8(self.simd_gt_u32x8(a0, b0), self.simd_gt_u32x8(a1, b1))
9379 }
9380 #[inline(always)]
9381 fn zip_low_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9382 let (a0, _) = self.split_u32x16(a);
9383 let (b0, _) = self.split_u32x16(b);
9384 self.combine_u32x8(self.zip_low_u32x8(a0, b0), self.zip_high_u32x8(a0, b0))
9385 }
9386 #[inline(always)]
9387 fn zip_high_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9388 let (_, a1) = self.split_u32x16(a);
9389 let (_, b1) = self.split_u32x16(b);
9390 self.combine_u32x8(self.zip_low_u32x8(a1, b1), self.zip_high_u32x8(a1, b1))
9391 }
9392 #[inline(always)]
9393 fn unzip_low_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9394 let (a0, a1) = self.split_u32x16(a);
9395 let (b0, b1) = self.split_u32x16(b);
9396 self.combine_u32x8(self.unzip_low_u32x8(a0, a1), self.unzip_low_u32x8(b0, b1))
9397 }
9398 #[inline(always)]
9399 fn unzip_high_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9400 let (a0, a1) = self.split_u32x16(a);
9401 let (b0, b1) = self.split_u32x16(b);
9402 self.combine_u32x8(self.unzip_high_u32x8(a0, a1), self.unzip_high_u32x8(b0, b1))
9403 }
9404 #[inline(always)]
9405 fn interleave_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> (u32x16<Self>, u32x16<Self>) {
9406 let (a0, a1) = self.split_u32x16(a);
9407 let (b0, b1) = self.split_u32x16(b);
9408 let lo_lo = self.zip_low_u32x8(a0, b0);
9409 let lo_hi = self.zip_high_u32x8(a0, b0);
9410 let hi_lo = self.zip_low_u32x8(a1, b1);
9411 let hi_hi = self.zip_high_u32x8(a1, b1);
9412 (
9413 self.combine_u32x8(lo_lo, lo_hi),
9414 self.combine_u32x8(hi_lo, hi_hi),
9415 )
9416 }
9417 #[inline(always)]
9418 fn deinterleave_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> (u32x16<Self>, u32x16<Self>) {
9419 let (a0, a1) = self.split_u32x16(a);
9420 let (b0, b1) = self.split_u32x16(b);
9421 let lo_even = self.unzip_low_u32x8(a0, a1);
9422 let lo_odd = self.unzip_high_u32x8(a0, a1);
9423 let hi_even = self.unzip_low_u32x8(b0, b1);
9424 let hi_odd = self.unzip_high_u32x8(b0, b1);
9425 (
9426 self.combine_u32x8(lo_even, hi_even),
9427 self.combine_u32x8(lo_odd, hi_odd),
9428 )
9429 }
9430 #[inline(always)]
9431 fn select_u32x16(self, a: mask32x16<Self>, b: u32x16<Self>, c: u32x16<Self>) -> u32x16<Self> {
9432 let (a0, a1) = self.split_mask32x16(a);
9433 let (b0, b1) = self.split_u32x16(b);
9434 let (c0, c1) = self.split_u32x16(c);
9435 self.combine_u32x8(self.select_u32x8(a0, b0, c0), self.select_u32x8(a1, b1, c1))
9436 }
9437 #[inline(always)]
9438 fn min_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9439 let (a0, a1) = self.split_u32x16(a);
9440 let (b0, b1) = self.split_u32x16(b);
9441 self.combine_u32x8(self.min_u32x8(a0, b0), self.min_u32x8(a1, b1))
9442 }
9443 #[inline(always)]
9444 fn max_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9445 let (a0, a1) = self.split_u32x16(a);
9446 let (b0, b1) = self.split_u32x16(b);
9447 self.combine_u32x8(self.max_u32x8(a0, b0), self.max_u32x8(a1, b1))
9448 }
9449 #[inline(always)]
9450 fn split_u32x16(self, a: u32x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
9451 (
9452 u32x8 {
9453 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9454 simd: self,
9455 },
9456 u32x8 {
9457 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9458 simd: self,
9459 },
9460 )
9461 }
9462 #[inline(always)]
9463 fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16<Self> {
9464 crate::kernel!(
9465 #[inline(always)]
9466 fn kernel(token: Sse4_2, src: &[u32; 16usize]) -> u32x16<Sse4_2> {
9467 let (chunks, []) = src.as_chunks::<4usize>() else {
9468 unreachable!()
9469 };
9470 let v0: __m128i =
9471 crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]);
9472 let v1: __m128i =
9473 crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]);
9474 let v2: __m128i =
9475 crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]);
9476 let v3: __m128i =
9477 crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]);
9478 let tmp0 = _mm_unpacklo_epi32(v0, v1);
9479 let tmp1 = _mm_unpackhi_epi32(v0, v1);
9480 let tmp2 = _mm_unpacklo_epi32(v2, v3);
9481 let tmp3 = _mm_unpackhi_epi32(v2, v3);
9482 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
9483 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
9484 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
9485 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
9486 token.combine_u32x8(
9487 token.combine_u32x4(out0.simd_into(token), out1.simd_into(token)),
9488 token.combine_u32x4(out2.simd_into(token), out3.simd_into(token)),
9489 )
9490 }
9491 );
9492 kernel(self, src)
9493 }
9494 #[inline(always)]
9495 fn store_interleaved_128_u32x16(self, a: u32x16<Self>, dest: &mut [u32; 16usize]) -> () {
9496 crate::kernel!(
9497 #[inline(always)]
9498 fn kernel(token: Sse4_2, a: u32x16<Sse4_2>, dest: &mut [u32; 16usize]) -> () {
9499 let (v01, v23) = token.split_u32x16(a);
9500 let (v0, v1) = token.split_u32x8(v01);
9501 let (v2, v3) = token.split_u32x8(v23);
9502 let v0 = v0.into();
9503 let v1 = v1.into();
9504 let v2 = v2.into();
9505 let v3 = v3.into();
9506 let tmp0 = _mm_unpacklo_epi32(v0, v1);
9507 let tmp1 = _mm_unpackhi_epi32(v0, v1);
9508 let tmp2 = _mm_unpacklo_epi32(v2, v3);
9509 let tmp3 = _mm_unpackhi_epi32(v2, v3);
9510 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
9511 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
9512 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
9513 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
9514 let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
9515 unreachable!()
9516 };
9517 crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
9518 out0,
9519 &mut chunks[0],
9520 );
9521 crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
9522 out1,
9523 &mut chunks[1],
9524 );
9525 crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
9526 out2,
9527 &mut chunks[2],
9528 );
9529 crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
9530 out3,
9531 &mut chunks[3],
9532 );
9533 }
9534 );
9535 kernel(self, a, dest);
9536 }
9537 #[inline(always)]
9538 fn reinterpret_u8_u32x16(self, a: u32x16<Self>) -> u8x64<Self> {
9539 let (a0, a1) = self.split_u32x16(a);
9540 self.combine_u8x32(self.reinterpret_u8_u32x8(a0), self.reinterpret_u8_u32x8(a1))
9541 }
9542 #[inline(always)]
9543 fn cvt_f32_u32x16(self, a: u32x16<Self>) -> f32x16<Self> {
9544 let (a0, a1) = self.split_u32x16(a);
9545 self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1))
9546 }
9547 #[inline(always)]
9548 fn splat_mask32x16(self, val: bool) -> mask32x16<Self> {
9549 let half = self.splat_mask32x8(val);
9550 self.combine_mask32x8(half, half)
9551 }
9552 #[inline(always)]
9553 fn load_array_mask32x16(self, val: [i32; 16usize]) -> mask32x16<Self> {
9554 mask32x16 {
9555 val: crate::transmute::checked_transmute_copy(&val),
9556 simd: self,
9557 }
9558 }
9559 #[inline(always)]
9560 fn as_array_mask32x16(self, a: mask32x16<Self>) -> [i32; 16usize] {
9561 crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i32; 16usize]>(&a.val.0)
9562 }
9563 #[inline(always)]
9564 fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16<Self> {
9565 let lo = self.from_bitmask_mask32x8(bits);
9566 let hi = self.from_bitmask_mask32x8(bits >> 8usize);
9567 self.combine_mask32x8(lo, hi)
9568 }
9569 #[inline(always)]
9570 fn to_bitmask_mask32x16(self, a: mask32x16<Self>) -> u64 {
9571 let (lo, hi) = self.split_mask32x16(a);
9572 let lo = self.to_bitmask_mask32x8(lo);
9573 let hi = self.to_bitmask_mask32x8(hi);
9574 lo | (hi << 8usize)
9575 }
9576 #[inline(always)]
9577 fn set_mask32x16(self, a: &mut mask32x16<Self>, index: usize, value: bool) -> () {
9578 assert!(
9579 index < 16usize,
9580 "mask lane index {index} is out of bounds for {} lanes",
9581 16usize
9582 );
9583 let mut lanes = self.as_array_mask32x16(*a);
9584 lanes[index] = if value { !0 } else { 0 };
9585 *a = self.load_array_mask32x16(lanes);
9586 }
9587 #[inline(always)]
9588 fn and_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask32x16<Self> {
9589 let (a0, a1) = self.split_mask32x16(a);
9590 let (b0, b1) = self.split_mask32x16(b);
9591 self.combine_mask32x8(self.and_mask32x8(a0, b0), self.and_mask32x8(a1, b1))
9592 }
9593 #[inline(always)]
9594 fn or_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask32x16<Self> {
9595 let (a0, a1) = self.split_mask32x16(a);
9596 let (b0, b1) = self.split_mask32x16(b);
9597 self.combine_mask32x8(self.or_mask32x8(a0, b0), self.or_mask32x8(a1, b1))
9598 }
9599 #[inline(always)]
9600 fn xor_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask32x16<Self> {
9601 let (a0, a1) = self.split_mask32x16(a);
9602 let (b0, b1) = self.split_mask32x16(b);
9603 self.combine_mask32x8(self.xor_mask32x8(a0, b0), self.xor_mask32x8(a1, b1))
9604 }
9605 #[inline(always)]
9606 fn not_mask32x16(self, a: mask32x16<Self>) -> mask32x16<Self> {
9607 let (a0, a1) = self.split_mask32x16(a);
9608 self.combine_mask32x8(self.not_mask32x8(a0), self.not_mask32x8(a1))
9609 }
9610 #[inline(always)]
9611 fn select_mask32x16(
9612 self,
9613 a: mask32x16<Self>,
9614 b: mask32x16<Self>,
9615 c: mask32x16<Self>,
9616 ) -> mask32x16<Self> {
9617 let (a0, a1) = self.split_mask32x16(a);
9618 let (b0, b1) = self.split_mask32x16(b);
9619 let (c0, c1) = self.split_mask32x16(c);
9620 self.combine_mask32x8(
9621 self.select_mask32x8(a0, b0, c0),
9622 self.select_mask32x8(a1, b1, c1),
9623 )
9624 }
9625 #[inline(always)]
9626 fn simd_eq_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask32x16<Self> {
9627 let (a0, a1) = self.split_mask32x16(a);
9628 let (b0, b1) = self.split_mask32x16(b);
9629 self.combine_mask32x8(self.simd_eq_mask32x8(a0, b0), self.simd_eq_mask32x8(a1, b1))
9630 }
9631 #[inline(always)]
9632 fn any_true_mask32x16(self, a: mask32x16<Self>) -> bool {
9633 let (a0, a1) = self.split_mask32x16(a);
9634 self.any_true_mask32x8(a0) || self.any_true_mask32x8(a1)
9635 }
9636 #[inline(always)]
9637 fn all_true_mask32x16(self, a: mask32x16<Self>) -> bool {
9638 let (a0, a1) = self.split_mask32x16(a);
9639 self.all_true_mask32x8(a0) && self.all_true_mask32x8(a1)
9640 }
9641 #[inline(always)]
9642 fn any_false_mask32x16(self, a: mask32x16<Self>) -> bool {
9643 let (a0, a1) = self.split_mask32x16(a);
9644 self.any_false_mask32x8(a0) || self.any_false_mask32x8(a1)
9645 }
9646 #[inline(always)]
9647 fn all_false_mask32x16(self, a: mask32x16<Self>) -> bool {
9648 let (a0, a1) = self.split_mask32x16(a);
9649 self.all_false_mask32x8(a0) && self.all_false_mask32x8(a1)
9650 }
9651 #[inline(always)]
9652 fn split_mask32x16(self, a: mask32x16<Self>) -> (mask32x8<Self>, mask32x8<Self>) {
9653 (
9654 mask32x8 {
9655 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9656 simd: self,
9657 },
9658 mask32x8 {
9659 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9660 simd: self,
9661 },
9662 )
9663 }
9664 #[inline(always)]
9665 fn splat_f64x8(self, val: f64) -> f64x8<Self> {
9666 let half = self.splat_f64x4(val);
9667 self.combine_f64x4(half, half)
9668 }
9669 #[inline(always)]
9670 fn load_array_f64x8(self, val: [f64; 8usize]) -> f64x8<Self> {
9671 f64x8 {
9672 val: crate::transmute::checked_transmute_copy(&val),
9673 simd: self,
9674 }
9675 }
9676 #[inline(always)]
9677 fn load_array_ref_f64x8(self, val: &[f64; 8usize]) -> f64x8<Self> {
9678 f64x8 {
9679 val: crate::transmute::checked_transmute_copy(val),
9680 simd: self,
9681 }
9682 }
9683 #[inline(always)]
9684 fn as_array_f64x8(self, a: f64x8<Self>) -> [f64; 8usize] {
9685 crate::transmute::checked_transmute_copy::<[__m128d; 4usize], [f64; 8usize]>(&a.val.0)
9686 }
9687 #[inline(always)]
9688 fn as_array_ref_f64x8(self, a: &f64x8<Self>) -> &[f64; 8usize] {
9689 crate::transmute::checked_cast_ref::<[__m128d; 4usize], [f64; 8usize]>(&a.val.0)
9690 }
9691 #[inline(always)]
9692 fn as_array_mut_f64x8(self, a: &mut f64x8<Self>) -> &mut [f64; 8usize] {
9693 crate::transmute::checked_cast_mut::<[__m128d; 4usize], [f64; 8usize]>(&mut a.val.0)
9694 }
9695 #[inline(always)]
9696 fn store_array_f64x8(self, a: f64x8<Self>, dest: &mut [f64; 8usize]) -> () {
9697 crate::transmute::checked_transmute_store(a.val.0, dest);
9698 }
9699 #[inline(always)]
9700 fn cvt_from_bytes_f64x8(self, a: u8x64<Self>) -> f64x8<Self> {
9701 f64x8 {
9702 val: crate::transmute::checked_transmute_copy(&a.val),
9703 simd: self,
9704 }
9705 }
9706 #[inline(always)]
9707 fn cvt_to_bytes_f64x8(self, a: f64x8<Self>) -> u8x64<Self> {
9708 u8x64 {
9709 val: crate::transmute::checked_transmute_copy(&a.val),
9710 simd: self,
9711 }
9712 }
9713 #[inline(always)]
9714 fn slide_f64x8<const SHIFT: usize>(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9715 if SHIFT >= 8usize {
9716 return b;
9717 }
9718 let result = cross_block_alignr_128x4(
9719 self,
9720 self.cvt_to_bytes_f64x8(b).val.0,
9721 self.cvt_to_bytes_f64x8(a).val.0,
9722 SHIFT * 8usize,
9723 );
9724 self.cvt_from_bytes_f64x8(u8x64 {
9725 val: crate::support::Aligned512(result),
9726 simd: self,
9727 })
9728 }
9729 #[inline(always)]
9730 fn slide_within_blocks_f64x8<const SHIFT: usize>(
9731 self,
9732 a: f64x8<Self>,
9733 b: f64x8<Self>,
9734 ) -> f64x8<Self> {
9735 let (a0, a1) = self.split_f64x8(a);
9736 let (b0, b1) = self.split_f64x8(b);
9737 self.combine_f64x4(
9738 self.slide_within_blocks_f64x4::<SHIFT>(a0, b0),
9739 self.slide_within_blocks_f64x4::<SHIFT>(a1, b1),
9740 )
9741 }
9742 #[inline(always)]
9743 fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8<Self>, indices: u8x64<Self>) -> f64x8<Self> {
9744 let (a0, a1) = self.split_f64x8(a);
9745 let (indices0, indices1) = self.split_u8x64(indices);
9746 self.combine_f64x4(
9747 self.swizzle_dyn_within_blocks_f64x4(a0, indices0),
9748 self.swizzle_dyn_within_blocks_f64x4(a1, indices1),
9749 )
9750 }
9751 #[inline(always)]
9752 fn abs_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
9753 let (a0, a1) = self.split_f64x8(a);
9754 self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1))
9755 }
9756 #[inline(always)]
9757 fn neg_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
9758 let (a0, a1) = self.split_f64x8(a);
9759 self.combine_f64x4(self.neg_f64x4(a0), self.neg_f64x4(a1))
9760 }
9761 #[inline(always)]
9762 fn sqrt_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
9763 let (a0, a1) = self.split_f64x8(a);
9764 self.combine_f64x4(self.sqrt_f64x4(a0), self.sqrt_f64x4(a1))
9765 }
9766 #[inline(always)]
9767 fn approximate_recip_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
9768 let (a0, a1) = self.split_f64x8(a);
9769 self.combine_f64x4(
9770 self.approximate_recip_f64x4(a0),
9771 self.approximate_recip_f64x4(a1),
9772 )
9773 }
9774 #[inline(always)]
9775 fn add_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9776 let (a0, a1) = self.split_f64x8(a);
9777 let (b0, b1) = self.split_f64x8(b);
9778 self.combine_f64x4(self.add_f64x4(a0, b0), self.add_f64x4(a1, b1))
9779 }
9780 #[inline(always)]
9781 fn sub_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9782 let (a0, a1) = self.split_f64x8(a);
9783 let (b0, b1) = self.split_f64x8(b);
9784 self.combine_f64x4(self.sub_f64x4(a0, b0), self.sub_f64x4(a1, b1))
9785 }
9786 #[inline(always)]
9787 fn mul_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9788 let (a0, a1) = self.split_f64x8(a);
9789 let (b0, b1) = self.split_f64x8(b);
9790 self.combine_f64x4(self.mul_f64x4(a0, b0), self.mul_f64x4(a1, b1))
9791 }
9792 #[inline(always)]
9793 fn div_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9794 let (a0, a1) = self.split_f64x8(a);
9795 let (b0, b1) = self.split_f64x8(b);
9796 self.combine_f64x4(self.div_f64x4(a0, b0), self.div_f64x4(a1, b1))
9797 }
9798 #[inline(always)]
9799 fn copysign_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9800 let (a0, a1) = self.split_f64x8(a);
9801 let (b0, b1) = self.split_f64x8(b);
9802 self.combine_f64x4(self.copysign_f64x4(a0, b0), self.copysign_f64x4(a1, b1))
9803 }
9804 #[inline(always)]
9805 fn simd_eq_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
9806 let (a0, a1) = self.split_f64x8(a);
9807 let (b0, b1) = self.split_f64x8(b);
9808 self.combine_mask64x4(self.simd_eq_f64x4(a0, b0), self.simd_eq_f64x4(a1, b1))
9809 }
9810 #[inline(always)]
9811 fn simd_lt_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
9812 let (a0, a1) = self.split_f64x8(a);
9813 let (b0, b1) = self.split_f64x8(b);
9814 self.combine_mask64x4(self.simd_lt_f64x4(a0, b0), self.simd_lt_f64x4(a1, b1))
9815 }
9816 #[inline(always)]
9817 fn simd_le_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
9818 let (a0, a1) = self.split_f64x8(a);
9819 let (b0, b1) = self.split_f64x8(b);
9820 self.combine_mask64x4(self.simd_le_f64x4(a0, b0), self.simd_le_f64x4(a1, b1))
9821 }
9822 #[inline(always)]
9823 fn simd_ge_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
9824 let (a0, a1) = self.split_f64x8(a);
9825 let (b0, b1) = self.split_f64x8(b);
9826 self.combine_mask64x4(self.simd_ge_f64x4(a0, b0), self.simd_ge_f64x4(a1, b1))
9827 }
9828 #[inline(always)]
9829 fn simd_gt_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
9830 let (a0, a1) = self.split_f64x8(a);
9831 let (b0, b1) = self.split_f64x8(b);
9832 self.combine_mask64x4(self.simd_gt_f64x4(a0, b0), self.simd_gt_f64x4(a1, b1))
9833 }
9834 #[inline(always)]
9835 fn zip_low_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9836 let (a0, _) = self.split_f64x8(a);
9837 let (b0, _) = self.split_f64x8(b);
9838 self.combine_f64x4(self.zip_low_f64x4(a0, b0), self.zip_high_f64x4(a0, b0))
9839 }
9840 #[inline(always)]
9841 fn zip_high_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9842 let (_, a1) = self.split_f64x8(a);
9843 let (_, b1) = self.split_f64x8(b);
9844 self.combine_f64x4(self.zip_low_f64x4(a1, b1), self.zip_high_f64x4(a1, b1))
9845 }
9846 #[inline(always)]
9847 fn unzip_low_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9848 let (a0, a1) = self.split_f64x8(a);
9849 let (b0, b1) = self.split_f64x8(b);
9850 self.combine_f64x4(self.unzip_low_f64x4(a0, a1), self.unzip_low_f64x4(b0, b1))
9851 }
9852 #[inline(always)]
9853 fn unzip_high_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9854 let (a0, a1) = self.split_f64x8(a);
9855 let (b0, b1) = self.split_f64x8(b);
9856 self.combine_f64x4(self.unzip_high_f64x4(a0, a1), self.unzip_high_f64x4(b0, b1))
9857 }
9858 #[inline(always)]
9859 fn interleave_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> (f64x8<Self>, f64x8<Self>) {
9860 let (a0, a1) = self.split_f64x8(a);
9861 let (b0, b1) = self.split_f64x8(b);
9862 let lo_lo = self.zip_low_f64x4(a0, b0);
9863 let lo_hi = self.zip_high_f64x4(a0, b0);
9864 let hi_lo = self.zip_low_f64x4(a1, b1);
9865 let hi_hi = self.zip_high_f64x4(a1, b1);
9866 (
9867 self.combine_f64x4(lo_lo, lo_hi),
9868 self.combine_f64x4(hi_lo, hi_hi),
9869 )
9870 }
9871 #[inline(always)]
9872 fn deinterleave_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> (f64x8<Self>, f64x8<Self>) {
9873 let (a0, a1) = self.split_f64x8(a);
9874 let (b0, b1) = self.split_f64x8(b);
9875 let lo_even = self.unzip_low_f64x4(a0, a1);
9876 let lo_odd = self.unzip_high_f64x4(a0, a1);
9877 let hi_even = self.unzip_low_f64x4(b0, b1);
9878 let hi_odd = self.unzip_high_f64x4(b0, b1);
9879 (
9880 self.combine_f64x4(lo_even, hi_even),
9881 self.combine_f64x4(lo_odd, hi_odd),
9882 )
9883 }
9884 #[inline(always)]
9885 fn max_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9886 let (a0, a1) = self.split_f64x8(a);
9887 let (b0, b1) = self.split_f64x8(b);
9888 self.combine_f64x4(self.max_f64x4(a0, b0), self.max_f64x4(a1, b1))
9889 }
9890 #[inline(always)]
9891 fn min_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9892 let (a0, a1) = self.split_f64x8(a);
9893 let (b0, b1) = self.split_f64x8(b);
9894 self.combine_f64x4(self.min_f64x4(a0, b0), self.min_f64x4(a1, b1))
9895 }
9896 #[inline(always)]
9897 fn max_precise_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9898 let (a0, a1) = self.split_f64x8(a);
9899 let (b0, b1) = self.split_f64x8(b);
9900 self.combine_f64x4(
9901 self.max_precise_f64x4(a0, b0),
9902 self.max_precise_f64x4(a1, b1),
9903 )
9904 }
9905 #[inline(always)]
9906 fn min_precise_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9907 let (a0, a1) = self.split_f64x8(a);
9908 let (b0, b1) = self.split_f64x8(b);
9909 self.combine_f64x4(
9910 self.min_precise_f64x4(a0, b0),
9911 self.min_precise_f64x4(a1, b1),
9912 )
9913 }
9914 #[inline(always)]
9915 fn mul_add_f64x8(self, a: f64x8<Self>, b: f64x8<Self>, c: f64x8<Self>) -> f64x8<Self> {
9916 let (a0, a1) = self.split_f64x8(a);
9917 let (b0, b1) = self.split_f64x8(b);
9918 let (c0, c1) = self.split_f64x8(c);
9919 self.combine_f64x4(
9920 self.mul_add_f64x4(a0, b0, c0),
9921 self.mul_add_f64x4(a1, b1, c1),
9922 )
9923 }
9924 #[inline(always)]
9925 fn mul_sub_f64x8(self, a: f64x8<Self>, b: f64x8<Self>, c: f64x8<Self>) -> f64x8<Self> {
9926 let (a0, a1) = self.split_f64x8(a);
9927 let (b0, b1) = self.split_f64x8(b);
9928 let (c0, c1) = self.split_f64x8(c);
9929 self.combine_f64x4(
9930 self.mul_sub_f64x4(a0, b0, c0),
9931 self.mul_sub_f64x4(a1, b1, c1),
9932 )
9933 }
9934 #[inline(always)]
9935 fn floor_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
9936 let (a0, a1) = self.split_f64x8(a);
9937 self.combine_f64x4(self.floor_f64x4(a0), self.floor_f64x4(a1))
9938 }
9939 #[inline(always)]
9940 fn ceil_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
9941 let (a0, a1) = self.split_f64x8(a);
9942 self.combine_f64x4(self.ceil_f64x4(a0), self.ceil_f64x4(a1))
9943 }
9944 #[inline(always)]
9945 fn round_ties_even_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
9946 let (a0, a1) = self.split_f64x8(a);
9947 self.combine_f64x4(
9948 self.round_ties_even_f64x4(a0),
9949 self.round_ties_even_f64x4(a1),
9950 )
9951 }
9952 #[inline(always)]
9953 fn fract_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
9954 let (a0, a1) = self.split_f64x8(a);
9955 self.combine_f64x4(self.fract_f64x4(a0), self.fract_f64x4(a1))
9956 }
9957 #[inline(always)]
9958 fn trunc_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
9959 let (a0, a1) = self.split_f64x8(a);
9960 self.combine_f64x4(self.trunc_f64x4(a0), self.trunc_f64x4(a1))
9961 }
9962 #[inline(always)]
9963 fn select_f64x8(self, a: mask64x8<Self>, b: f64x8<Self>, c: f64x8<Self>) -> f64x8<Self> {
9964 let (a0, a1) = self.split_mask64x8(a);
9965 let (b0, b1) = self.split_f64x8(b);
9966 let (c0, c1) = self.split_f64x8(c);
9967 self.combine_f64x4(self.select_f64x4(a0, b0, c0), self.select_f64x4(a1, b1, c1))
9968 }
9969 #[inline(always)]
9970 fn split_f64x8(self, a: f64x8<Self>) -> (f64x4<Self>, f64x4<Self>) {
9971 (
9972 f64x4 {
9973 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9974 simd: self,
9975 },
9976 f64x4 {
9977 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9978 simd: self,
9979 },
9980 )
9981 }
9982 #[inline(always)]
9983 fn reinterpret_f32_f64x8(self, a: f64x8<Self>) -> f32x16<Self> {
9984 let (a0, a1) = self.split_f64x8(a);
9985 self.combine_f32x8(
9986 self.reinterpret_f32_f64x4(a0),
9987 self.reinterpret_f32_f64x4(a1),
9988 )
9989 }
9990 #[inline(always)]
9991 fn splat_mask64x8(self, val: bool) -> mask64x8<Self> {
9992 let half = self.splat_mask64x4(val);
9993 self.combine_mask64x4(half, half)
9994 }
9995 #[inline(always)]
9996 fn load_array_mask64x8(self, val: [i64; 8usize]) -> mask64x8<Self> {
9997 mask64x8 {
9998 val: crate::transmute::checked_transmute_copy(&val),
9999 simd: self,
10000 }
10001 }
10002 #[inline(always)]
10003 fn as_array_mask64x8(self, a: mask64x8<Self>) -> [i64; 8usize] {
10004 crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [i64; 8usize]>(&a.val.0)
10005 }
10006 #[inline(always)]
10007 fn from_bitmask_mask64x8(self, bits: u64) -> mask64x8<Self> {
10008 let lo = self.from_bitmask_mask64x4(bits);
10009 let hi = self.from_bitmask_mask64x4(bits >> 4usize);
10010 self.combine_mask64x4(lo, hi)
10011 }
10012 #[inline(always)]
10013 fn to_bitmask_mask64x8(self, a: mask64x8<Self>) -> u64 {
10014 let (lo, hi) = self.split_mask64x8(a);
10015 let lo = self.to_bitmask_mask64x4(lo);
10016 let hi = self.to_bitmask_mask64x4(hi);
10017 lo | (hi << 4usize)
10018 }
10019 #[inline(always)]
10020 fn set_mask64x8(self, a: &mut mask64x8<Self>, index: usize, value: bool) -> () {
10021 assert!(
10022 index < 8usize,
10023 "mask lane index {index} is out of bounds for {} lanes",
10024 8usize
10025 );
10026 let mut lanes = self.as_array_mask64x8(*a);
10027 lanes[index] = if value { !0 } else { 0 };
10028 *a = self.load_array_mask64x8(lanes);
10029 }
10030 #[inline(always)]
10031 fn and_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask64x8<Self> {
10032 let (a0, a1) = self.split_mask64x8(a);
10033 let (b0, b1) = self.split_mask64x8(b);
10034 self.combine_mask64x4(self.and_mask64x4(a0, b0), self.and_mask64x4(a1, b1))
10035 }
10036 #[inline(always)]
10037 fn or_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask64x8<Self> {
10038 let (a0, a1) = self.split_mask64x8(a);
10039 let (b0, b1) = self.split_mask64x8(b);
10040 self.combine_mask64x4(self.or_mask64x4(a0, b0), self.or_mask64x4(a1, b1))
10041 }
10042 #[inline(always)]
10043 fn xor_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask64x8<Self> {
10044 let (a0, a1) = self.split_mask64x8(a);
10045 let (b0, b1) = self.split_mask64x8(b);
10046 self.combine_mask64x4(self.xor_mask64x4(a0, b0), self.xor_mask64x4(a1, b1))
10047 }
10048 #[inline(always)]
10049 fn not_mask64x8(self, a: mask64x8<Self>) -> mask64x8<Self> {
10050 let (a0, a1) = self.split_mask64x8(a);
10051 self.combine_mask64x4(self.not_mask64x4(a0), self.not_mask64x4(a1))
10052 }
10053 #[inline(always)]
10054 fn select_mask64x8(
10055 self,
10056 a: mask64x8<Self>,
10057 b: mask64x8<Self>,
10058 c: mask64x8<Self>,
10059 ) -> mask64x8<Self> {
10060 let (a0, a1) = self.split_mask64x8(a);
10061 let (b0, b1) = self.split_mask64x8(b);
10062 let (c0, c1) = self.split_mask64x8(c);
10063 self.combine_mask64x4(
10064 self.select_mask64x4(a0, b0, c0),
10065 self.select_mask64x4(a1, b1, c1),
10066 )
10067 }
10068 #[inline(always)]
10069 fn simd_eq_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask64x8<Self> {
10070 let (a0, a1) = self.split_mask64x8(a);
10071 let (b0, b1) = self.split_mask64x8(b);
10072 self.combine_mask64x4(self.simd_eq_mask64x4(a0, b0), self.simd_eq_mask64x4(a1, b1))
10073 }
10074 #[inline(always)]
10075 fn any_true_mask64x8(self, a: mask64x8<Self>) -> bool {
10076 let (a0, a1) = self.split_mask64x8(a);
10077 self.any_true_mask64x4(a0) || self.any_true_mask64x4(a1)
10078 }
10079 #[inline(always)]
10080 fn all_true_mask64x8(self, a: mask64x8<Self>) -> bool {
10081 let (a0, a1) = self.split_mask64x8(a);
10082 self.all_true_mask64x4(a0) && self.all_true_mask64x4(a1)
10083 }
10084 #[inline(always)]
10085 fn any_false_mask64x8(self, a: mask64x8<Self>) -> bool {
10086 let (a0, a1) = self.split_mask64x8(a);
10087 self.any_false_mask64x4(a0) || self.any_false_mask64x4(a1)
10088 }
10089 #[inline(always)]
10090 fn all_false_mask64x8(self, a: mask64x8<Self>) -> bool {
10091 let (a0, a1) = self.split_mask64x8(a);
10092 self.all_false_mask64x4(a0) && self.all_false_mask64x4(a1)
10093 }
10094 #[inline(always)]
10095 fn split_mask64x8(self, a: mask64x8<Self>) -> (mask64x4<Self>, mask64x4<Self>) {
10096 (
10097 mask64x4 {
10098 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10099 simd: self,
10100 },
10101 mask64x4 {
10102 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10103 simd: self,
10104 },
10105 )
10106 }
10107}
10108impl<S: Simd> SimdFrom<__m128, S> for f32x4<S> {
10109 #[inline(always)]
10110 fn simd_from(simd: S, arch: __m128) -> Self {
10111 Self {
10112 val: crate::transmute::checked_transmute_copy(&arch),
10113 simd,
10114 }
10115 }
10116}
10117impl<S: Simd> From<f32x4<S>> for __m128 {
10118 #[inline(always)]
10119 fn from(value: f32x4<S>) -> Self {
10120 crate::transmute::checked_transmute_copy(&value.val)
10121 }
10122}
10123impl<S: Simd> SimdFrom<__m128i, S> for i8x16<S> {
10124 #[inline(always)]
10125 fn simd_from(simd: S, arch: __m128i) -> Self {
10126 Self {
10127 val: crate::transmute::checked_transmute_copy(&arch),
10128 simd,
10129 }
10130 }
10131}
10132impl<S: Simd> From<i8x16<S>> for __m128i {
10133 #[inline(always)]
10134 fn from(value: i8x16<S>) -> Self {
10135 crate::transmute::checked_transmute_copy(&value.val)
10136 }
10137}
10138impl<S: Simd> SimdFrom<__m128i, S> for u8x16<S> {
10139 #[inline(always)]
10140 fn simd_from(simd: S, arch: __m128i) -> Self {
10141 Self {
10142 val: crate::transmute::checked_transmute_copy(&arch),
10143 simd,
10144 }
10145 }
10146}
10147impl<S: Simd> From<u8x16<S>> for __m128i {
10148 #[inline(always)]
10149 fn from(value: u8x16<S>) -> Self {
10150 crate::transmute::checked_transmute_copy(&value.val)
10151 }
10152}
10153impl<S: Simd> SimdFrom<__m128i, S> for mask8x16<S> {
10154 #[inline(always)]
10155 fn simd_from(simd: S, arch: __m128i) -> Self {
10156 let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&arch);
10157 lanes.simd_into(simd)
10158 }
10159}
10160impl<S: Simd> From<mask8x16<S>> for __m128i {
10161 #[inline(always)]
10162 fn from(value: mask8x16<S>) -> Self {
10163 let lanes: [i8; 16usize] = value.into();
10164 crate::transmute::checked_transmute_copy(&lanes)
10165 }
10166}
10167impl<S: Simd> SimdFrom<__m128i, S> for i16x8<S> {
10168 #[inline(always)]
10169 fn simd_from(simd: S, arch: __m128i) -> Self {
10170 Self {
10171 val: crate::transmute::checked_transmute_copy(&arch),
10172 simd,
10173 }
10174 }
10175}
10176impl<S: Simd> From<i16x8<S>> for __m128i {
10177 #[inline(always)]
10178 fn from(value: i16x8<S>) -> Self {
10179 crate::transmute::checked_transmute_copy(&value.val)
10180 }
10181}
10182impl<S: Simd> SimdFrom<__m128i, S> for u16x8<S> {
10183 #[inline(always)]
10184 fn simd_from(simd: S, arch: __m128i) -> Self {
10185 Self {
10186 val: crate::transmute::checked_transmute_copy(&arch),
10187 simd,
10188 }
10189 }
10190}
10191impl<S: Simd> From<u16x8<S>> for __m128i {
10192 #[inline(always)]
10193 fn from(value: u16x8<S>) -> Self {
10194 crate::transmute::checked_transmute_copy(&value.val)
10195 }
10196}
10197impl<S: Simd> SimdFrom<__m128i, S> for mask16x8<S> {
10198 #[inline(always)]
10199 fn simd_from(simd: S, arch: __m128i) -> Self {
10200 let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&arch);
10201 lanes.simd_into(simd)
10202 }
10203}
10204impl<S: Simd> From<mask16x8<S>> for __m128i {
10205 #[inline(always)]
10206 fn from(value: mask16x8<S>) -> Self {
10207 let lanes: [i16; 8usize] = value.into();
10208 crate::transmute::checked_transmute_copy(&lanes)
10209 }
10210}
10211impl<S: Simd> SimdFrom<__m128i, S> for i32x4<S> {
10212 #[inline(always)]
10213 fn simd_from(simd: S, arch: __m128i) -> Self {
10214 Self {
10215 val: crate::transmute::checked_transmute_copy(&arch),
10216 simd,
10217 }
10218 }
10219}
10220impl<S: Simd> From<i32x4<S>> for __m128i {
10221 #[inline(always)]
10222 fn from(value: i32x4<S>) -> Self {
10223 crate::transmute::checked_transmute_copy(&value.val)
10224 }
10225}
10226impl<S: Simd> SimdFrom<__m128i, S> for u32x4<S> {
10227 #[inline(always)]
10228 fn simd_from(simd: S, arch: __m128i) -> Self {
10229 Self {
10230 val: crate::transmute::checked_transmute_copy(&arch),
10231 simd,
10232 }
10233 }
10234}
10235impl<S: Simd> From<u32x4<S>> for __m128i {
10236 #[inline(always)]
10237 fn from(value: u32x4<S>) -> Self {
10238 crate::transmute::checked_transmute_copy(&value.val)
10239 }
10240}
10241impl<S: Simd> SimdFrom<__m128i, S> for mask32x4<S> {
10242 #[inline(always)]
10243 fn simd_from(simd: S, arch: __m128i) -> Self {
10244 let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&arch);
10245 lanes.simd_into(simd)
10246 }
10247}
10248impl<S: Simd> From<mask32x4<S>> for __m128i {
10249 #[inline(always)]
10250 fn from(value: mask32x4<S>) -> Self {
10251 let lanes: [i32; 4usize] = value.into();
10252 crate::transmute::checked_transmute_copy(&lanes)
10253 }
10254}
10255impl<S: Simd> SimdFrom<__m128d, S> for f64x2<S> {
10256 #[inline(always)]
10257 fn simd_from(simd: S, arch: __m128d) -> Self {
10258 Self {
10259 val: crate::transmute::checked_transmute_copy(&arch),
10260 simd,
10261 }
10262 }
10263}
10264impl<S: Simd> From<f64x2<S>> for __m128d {
10265 #[inline(always)]
10266 fn from(value: f64x2<S>) -> Self {
10267 crate::transmute::checked_transmute_copy(&value.val)
10268 }
10269}
10270impl<S: Simd> SimdFrom<__m128i, S> for mask64x2<S> {
10271 #[inline(always)]
10272 fn simd_from(simd: S, arch: __m128i) -> Self {
10273 let lanes: [i64; 2usize] = crate::transmute::checked_transmute_copy(&arch);
10274 lanes.simd_into(simd)
10275 }
10276}
10277impl<S: Simd> From<mask64x2<S>> for __m128i {
10278 #[inline(always)]
10279 fn from(value: mask64x2<S>) -> Self {
10280 let lanes: [i64; 2usize] = value.into();
10281 crate::transmute::checked_transmute_copy(&lanes)
10282 }
10283}
10284crate::kernel!(
10285 #[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"]
10286 #[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
10287 #[doc = r" Rust doesn't currently let you do math on const generics."]
10288 #[inline(always)]
10289 fn dyn_alignr_128(token: Sse4_2, a: __m128i, b: __m128i, shift: usize) -> __m128i {
10290 match shift {
10291 0usize => _mm_alignr_epi8::<0i32>(a, b),
10292 1usize => _mm_alignr_epi8::<1i32>(a, b),
10293 2usize => _mm_alignr_epi8::<2i32>(a, b),
10294 3usize => _mm_alignr_epi8::<3i32>(a, b),
10295 4usize => _mm_alignr_epi8::<4i32>(a, b),
10296 5usize => _mm_alignr_epi8::<5i32>(a, b),
10297 6usize => _mm_alignr_epi8::<6i32>(a, b),
10298 7usize => _mm_alignr_epi8::<7i32>(a, b),
10299 8usize => _mm_alignr_epi8::<8i32>(a, b),
10300 9usize => _mm_alignr_epi8::<9i32>(a, b),
10301 10usize => _mm_alignr_epi8::<10i32>(a, b),
10302 11usize => _mm_alignr_epi8::<11i32>(a, b),
10303 12usize => _mm_alignr_epi8::<12i32>(a, b),
10304 13usize => _mm_alignr_epi8::<13i32>(a, b),
10305 14usize => _mm_alignr_epi8::<14i32>(a, b),
10306 15usize => _mm_alignr_epi8::<15i32>(a, b),
10307 _ => unreachable!(),
10308 }
10309 }
10310);
10311crate::kernel!(
10312 #[doc = r" Concatenates `b` and `a` (each N blocks) and extracts N blocks starting at byte offset `shift_bytes`."]
10313 #[doc = r" Extracts from [b : a] (b in low bytes, a in high bytes), matching `alignr` semantics."]
10314 #[inline(always)]
10315 fn cross_block_alignr_128x2(
10316 token: Sse4_2,
10317 a: [__m128i; 2usize],
10318 b: [__m128i; 2usize],
10319 shift_bytes: usize,
10320 ) -> [__m128i; 2usize] {
10321 [
10322 {
10323 let [lo, hi] =
10324 crate::support::cross_block_slide_blocks_at(&b, &a, 0usize, shift_bytes);
10325 dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10326 },
10327 {
10328 let [lo, hi] =
10329 crate::support::cross_block_slide_blocks_at(&b, &a, 1usize, shift_bytes);
10330 dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10331 },
10332 ]
10333 }
10334);
10335crate::kernel!(
10336 #[doc = r" Concatenates `b` and `a` (each N blocks) and extracts N blocks starting at byte offset `shift_bytes`."]
10337 #[doc = r" Extracts from [b : a] (b in low bytes, a in high bytes), matching `alignr` semantics."]
10338 #[inline(always)]
10339 fn cross_block_alignr_128x4(
10340 token: Sse4_2,
10341 a: [__m128i; 4usize],
10342 b: [__m128i; 4usize],
10343 shift_bytes: usize,
10344 ) -> [__m128i; 4usize] {
10345 [
10346 {
10347 let [lo, hi] =
10348 crate::support::cross_block_slide_blocks_at(&b, &a, 0usize, shift_bytes);
10349 dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10350 },
10351 {
10352 let [lo, hi] =
10353 crate::support::cross_block_slide_blocks_at(&b, &a, 1usize, shift_bytes);
10354 dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10355 },
10356 {
10357 let [lo, hi] =
10358 crate::support::cross_block_slide_blocks_at(&b, &a, 2usize, shift_bytes);
10359 dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10360 },
10361 {
10362 let [lo, hi] =
10363 crate::support::cross_block_slide_blocks_at(&b, &a, 3usize, shift_bytes);
10364 dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10365 },
10366 ]
10367 }
10368);