1#![allow(
7 clippy::identity_op,
8 reason = "AVX-512 mask code is generated uniformly for all __mmask widths"
9)]
10#![allow(
11 clippy::useless_conversion,
12 reason = "AVX-512 mask code is generated uniformly for all __mmask widths"
13)]
14use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal};
15use crate::{
16 f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4,
17 i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16,
18 mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64,
19 u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8,
20};
21#[cfg(target_arch = "x86")]
22use core::arch::x86::*;
23#[cfg(target_arch = "x86_64")]
24use core::arch::x86_64::*;
25use core::ops::*;
26#[doc = "A token for AVX-512 intrinsics on `x86` and `x86_64`, representing an Ice Lake feature level."]
27#[doc = "# Browsing the documentation"]
28#[doc = "The method list on this struct is very verbose."]
29#[doc = "A better way to browse the docs is by looking at vector types such as [`u32x4`], [`f32x4`] or [`mask32x4`]."]
30#[doc = "They include all the operations listed here, and also provide some additional convenience methods."]
31#[derive(Clone, Copy, Debug)]
32pub struct Avx512 {
33 _private: (),
34}
35impl Avx512 {
36 #[doc = "Create a SIMD token proving that the Ice Lake AVX-512 features are available."]
37 #[doc = r""]
38 #[doc = r" Most users should safely obtain the token from [`Level::new`] instead of calling this function."]
39 #[doc = r""]
40 #[doc = r" This function can be called without an `unsafe` block from a function"]
41 #[doc = r" with all the required target features enabled via the `#[target_feature]` annotation."]
42 #[doc = r""]
43 #[doc = r" # Safety"]
44 #[doc = r""]
45 #[doc = "When invoking this function through an `unsafe` block, the caller must ensure that the current CPU supports `adx`, `aes`, `avx512bitalg`, `avx512bw`, `avx512cd`, `avx512dq`, `avx512f`, `avx512ifma`, `avx512vbmi`, `avx512vbmi2`, `avx512vl`, `avx512vnni`, `avx512vpopcntdq`, `bmi1`, `bmi2`, `cmpxchg16b`, `fma`, `fxsr`, `gfni`, `lzcnt`, `movbe`, `pclmulqdq`, `popcnt`, `rdrand`, `rdseed`, `sha`, `vaes`, `vpclmulqdq`, `xsave`, `xsavec`, `xsaveopt`, `xsaves`."]
46 #[inline]
47 #[target_feature(
48 enable = "adx,aes,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512f,avx512ifma,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vpopcntdq,bmi1,bmi2,cmpxchg16b,fma,fxsr,gfni,lzcnt,movbe,pclmulqdq,popcnt,rdrand,rdseed,sha,vaes,vpclmulqdq,xsave,xsavec,xsaveopt,xsaves"
49 )]
50 pub const fn assume_supported() -> Self {
51 Self { _private: () }
52 }
53}
54impl Seal for Avx512 {}
55impl ExtractToken for Avx512 {
56 type S = Self;
57 #[inline]
58 fn token(&self) -> Self {
59 *self
60 }
61}
62impl ArchTypes for Avx512 {
63 type f32x4 = crate::support::Aligned128<__m128>;
64 type i8x16 = crate::support::Aligned128<__m128i>;
65 type u8x16 = crate::support::Aligned128<__m128i>;
66 type mask8x16 = __mmask16;
67 type i16x8 = crate::support::Aligned128<__m128i>;
68 type u16x8 = crate::support::Aligned128<__m128i>;
69 type mask16x8 = __mmask8;
70 type i32x4 = crate::support::Aligned128<__m128i>;
71 type u32x4 = crate::support::Aligned128<__m128i>;
72 type mask32x4 = __mmask8;
73 type f64x2 = crate::support::Aligned128<__m128d>;
74 type i64x2 = crate::support::Aligned128<__m128i>;
75 type u64x2 = crate::support::Aligned128<__m128i>;
76 type mask64x2 = __mmask8;
77 type f32x8 = crate::support::Aligned256<__m256>;
78 type i8x32 = crate::support::Aligned256<__m256i>;
79 type u8x32 = crate::support::Aligned256<__m256i>;
80 type mask8x32 = __mmask32;
81 type i16x16 = crate::support::Aligned256<__m256i>;
82 type u16x16 = crate::support::Aligned256<__m256i>;
83 type mask16x16 = __mmask16;
84 type i32x8 = crate::support::Aligned256<__m256i>;
85 type u32x8 = crate::support::Aligned256<__m256i>;
86 type mask32x8 = __mmask8;
87 type f64x4 = crate::support::Aligned256<__m256d>;
88 type i64x4 = crate::support::Aligned256<__m256i>;
89 type u64x4 = crate::support::Aligned256<__m256i>;
90 type mask64x4 = __mmask8;
91 type f32x16 = crate::support::Aligned512<__m512>;
92 type i8x64 = crate::support::Aligned512<__m512i>;
93 type u8x64 = crate::support::Aligned512<__m512i>;
94 type mask8x64 = __mmask64;
95 type i16x32 = crate::support::Aligned512<__m512i>;
96 type u16x32 = crate::support::Aligned512<__m512i>;
97 type mask16x32 = __mmask32;
98 type i32x16 = crate::support::Aligned512<__m512i>;
99 type u32x16 = crate::support::Aligned512<__m512i>;
100 type mask32x16 = __mmask16;
101 type f64x8 = crate::support::Aligned512<__m512d>;
102 type i64x8 = crate::support::Aligned512<__m512i>;
103 type u64x8 = crate::support::Aligned512<__m512i>;
104 type mask64x8 = __mmask8;
105 #[inline(always)]
106 fn mask8x16_from_array(self, val: [i8; 16usize]) -> Self::mask8x16 {
107 crate::kernel!(
108 #[inline(always)]
109 fn kernel(_token: Avx512, val: [i8; 16usize]) -> __mmask16 {
110 let lanes = crate::transmute::checked_transmute_copy(&val);
111 _mm_movepi8_mask(lanes)
112 }
113 );
114 kernel(self, val)
115 }
116 #[inline(always)]
117 fn mask8x16_to_array(self, val: Self::mask8x16) -> [i8; 16usize] {
118 crate::kernel!(
119 #[inline(always)]
120 fn kernel(_token: Avx512, val: __mmask16) -> [i8; 16usize] {
121 let lanes = _mm_movm_epi8(val);
122 crate::transmute::checked_transmute_copy(&lanes)
123 }
124 );
125 kernel(self, val)
126 }
127 #[inline(always)]
128 fn mask16x8_from_array(self, val: [i16; 8usize]) -> Self::mask16x8 {
129 crate::kernel!(
130 #[inline(always)]
131 fn kernel(_token: Avx512, val: [i16; 8usize]) -> __mmask8 {
132 let lanes = crate::transmute::checked_transmute_copy(&val);
133 _mm_movepi16_mask(lanes)
134 }
135 );
136 kernel(self, val)
137 }
138 #[inline(always)]
139 fn mask16x8_to_array(self, val: Self::mask16x8) -> [i16; 8usize] {
140 crate::kernel!(
141 #[inline(always)]
142 fn kernel(_token: Avx512, val: __mmask8) -> [i16; 8usize] {
143 let lanes = _mm_movm_epi16(val);
144 crate::transmute::checked_transmute_copy(&lanes)
145 }
146 );
147 kernel(self, val)
148 }
149 #[inline(always)]
150 fn mask32x4_from_array(self, val: [i32; 4usize]) -> Self::mask32x4 {
151 crate::kernel!(
152 #[inline(always)]
153 fn kernel(_token: Avx512, val: [i32; 4usize]) -> __mmask8 {
154 let lanes = crate::transmute::checked_transmute_copy(&val);
155 _mm_movepi32_mask(lanes)
156 }
157 );
158 kernel(self, val)
159 }
160 #[inline(always)]
161 fn mask32x4_to_array(self, val: Self::mask32x4) -> [i32; 4usize] {
162 crate::kernel!(
163 #[inline(always)]
164 fn kernel(_token: Avx512, val: __mmask8) -> [i32; 4usize] {
165 let lanes = _mm_movm_epi32(val);
166 crate::transmute::checked_transmute_copy(&lanes)
167 }
168 );
169 kernel(self, val)
170 }
171 #[inline(always)]
172 fn mask64x2_from_array(self, val: [i64; 2usize]) -> Self::mask64x2 {
173 crate::kernel!(
174 #[inline(always)]
175 fn kernel(_token: Avx512, val: [i64; 2usize]) -> __mmask8 {
176 let lanes = crate::transmute::checked_transmute_copy(&val);
177 _mm_movepi64_mask(lanes)
178 }
179 );
180 kernel(self, val)
181 }
182 #[inline(always)]
183 fn mask64x2_to_array(self, val: Self::mask64x2) -> [i64; 2usize] {
184 crate::kernel!(
185 #[inline(always)]
186 fn kernel(_token: Avx512, val: __mmask8) -> [i64; 2usize] {
187 let lanes = _mm_movm_epi64(val);
188 crate::transmute::checked_transmute_copy(&lanes)
189 }
190 );
191 kernel(self, val)
192 }
193 #[inline(always)]
194 fn mask8x32_from_array(self, val: [i8; 32usize]) -> Self::mask8x32 {
195 crate::kernel!(
196 #[inline(always)]
197 fn kernel(_token: Avx512, val: [i8; 32usize]) -> __mmask32 {
198 let lanes = crate::transmute::checked_transmute_copy(&val);
199 _mm256_movepi8_mask(lanes)
200 }
201 );
202 kernel(self, val)
203 }
204 #[inline(always)]
205 fn mask8x32_to_array(self, val: Self::mask8x32) -> [i8; 32usize] {
206 crate::kernel!(
207 #[inline(always)]
208 fn kernel(_token: Avx512, val: __mmask32) -> [i8; 32usize] {
209 let lanes = _mm256_movm_epi8(val);
210 crate::transmute::checked_transmute_copy(&lanes)
211 }
212 );
213 kernel(self, val)
214 }
215 #[inline(always)]
216 fn mask16x16_from_array(self, val: [i16; 16usize]) -> Self::mask16x16 {
217 crate::kernel!(
218 #[inline(always)]
219 fn kernel(_token: Avx512, val: [i16; 16usize]) -> __mmask16 {
220 let lanes = crate::transmute::checked_transmute_copy(&val);
221 _mm256_movepi16_mask(lanes)
222 }
223 );
224 kernel(self, val)
225 }
226 #[inline(always)]
227 fn mask16x16_to_array(self, val: Self::mask16x16) -> [i16; 16usize] {
228 crate::kernel!(
229 #[inline(always)]
230 fn kernel(_token: Avx512, val: __mmask16) -> [i16; 16usize] {
231 let lanes = _mm256_movm_epi16(val);
232 crate::transmute::checked_transmute_copy(&lanes)
233 }
234 );
235 kernel(self, val)
236 }
237 #[inline(always)]
238 fn mask32x8_from_array(self, val: [i32; 8usize]) -> Self::mask32x8 {
239 crate::kernel!(
240 #[inline(always)]
241 fn kernel(_token: Avx512, val: [i32; 8usize]) -> __mmask8 {
242 let lanes = crate::transmute::checked_transmute_copy(&val);
243 _mm256_movepi32_mask(lanes)
244 }
245 );
246 kernel(self, val)
247 }
248 #[inline(always)]
249 fn mask32x8_to_array(self, val: Self::mask32x8) -> [i32; 8usize] {
250 crate::kernel!(
251 #[inline(always)]
252 fn kernel(_token: Avx512, val: __mmask8) -> [i32; 8usize] {
253 let lanes = _mm256_movm_epi32(val);
254 crate::transmute::checked_transmute_copy(&lanes)
255 }
256 );
257 kernel(self, val)
258 }
259 #[inline(always)]
260 fn mask64x4_from_array(self, val: [i64; 4usize]) -> Self::mask64x4 {
261 crate::kernel!(
262 #[inline(always)]
263 fn kernel(_token: Avx512, val: [i64; 4usize]) -> __mmask8 {
264 let lanes = crate::transmute::checked_transmute_copy(&val);
265 _mm256_movepi64_mask(lanes)
266 }
267 );
268 kernel(self, val)
269 }
270 #[inline(always)]
271 fn mask64x4_to_array(self, val: Self::mask64x4) -> [i64; 4usize] {
272 crate::kernel!(
273 #[inline(always)]
274 fn kernel(_token: Avx512, val: __mmask8) -> [i64; 4usize] {
275 let lanes = _mm256_movm_epi64(val);
276 crate::transmute::checked_transmute_copy(&lanes)
277 }
278 );
279 kernel(self, val)
280 }
281 #[inline(always)]
282 fn mask8x64_from_array(self, val: [i8; 64usize]) -> Self::mask8x64 {
283 crate::kernel!(
284 #[inline(always)]
285 fn kernel(_token: Avx512, val: [i8; 64usize]) -> __mmask64 {
286 let lanes = crate::transmute::checked_transmute_copy(&val);
287 _mm512_movepi8_mask(lanes)
288 }
289 );
290 kernel(self, val)
291 }
292 #[inline(always)]
293 fn mask8x64_to_array(self, val: Self::mask8x64) -> [i8; 64usize] {
294 crate::kernel!(
295 #[inline(always)]
296 fn kernel(_token: Avx512, val: __mmask64) -> [i8; 64usize] {
297 let lanes = _mm512_movm_epi8(val);
298 crate::transmute::checked_transmute_copy(&lanes)
299 }
300 );
301 kernel(self, val)
302 }
303 #[inline(always)]
304 fn mask16x32_from_array(self, val: [i16; 32usize]) -> Self::mask16x32 {
305 crate::kernel!(
306 #[inline(always)]
307 fn kernel(_token: Avx512, val: [i16; 32usize]) -> __mmask32 {
308 let lanes = crate::transmute::checked_transmute_copy(&val);
309 _mm512_movepi16_mask(lanes)
310 }
311 );
312 kernel(self, val)
313 }
314 #[inline(always)]
315 fn mask16x32_to_array(self, val: Self::mask16x32) -> [i16; 32usize] {
316 crate::kernel!(
317 #[inline(always)]
318 fn kernel(_token: Avx512, val: __mmask32) -> [i16; 32usize] {
319 let lanes = _mm512_movm_epi16(val);
320 crate::transmute::checked_transmute_copy(&lanes)
321 }
322 );
323 kernel(self, val)
324 }
325 #[inline(always)]
326 fn mask32x16_from_array(self, val: [i32; 16usize]) -> Self::mask32x16 {
327 crate::kernel!(
328 #[inline(always)]
329 fn kernel(_token: Avx512, val: [i32; 16usize]) -> __mmask16 {
330 let lanes = crate::transmute::checked_transmute_copy(&val);
331 _mm512_movepi32_mask(lanes)
332 }
333 );
334 kernel(self, val)
335 }
336 #[inline(always)]
337 fn mask32x16_to_array(self, val: Self::mask32x16) -> [i32; 16usize] {
338 crate::kernel!(
339 #[inline(always)]
340 fn kernel(_token: Avx512, val: __mmask16) -> [i32; 16usize] {
341 let lanes = _mm512_movm_epi32(val);
342 crate::transmute::checked_transmute_copy(&lanes)
343 }
344 );
345 kernel(self, val)
346 }
347 #[inline(always)]
348 fn mask64x8_from_array(self, val: [i64; 8usize]) -> Self::mask64x8 {
349 crate::kernel!(
350 #[inline(always)]
351 fn kernel(_token: Avx512, val: [i64; 8usize]) -> __mmask8 {
352 let lanes = crate::transmute::checked_transmute_copy(&val);
353 _mm512_movepi64_mask(lanes)
354 }
355 );
356 kernel(self, val)
357 }
358 #[inline(always)]
359 fn mask64x8_to_array(self, val: Self::mask64x8) -> [i64; 8usize] {
360 crate::kernel!(
361 #[inline(always)]
362 fn kernel(_token: Avx512, val: __mmask8) -> [i64; 8usize] {
363 let lanes = _mm512_movm_epi64(val);
364 crate::transmute::checked_transmute_copy(&lanes)
365 }
366 );
367 kernel(self, val)
368 }
369}
370impl Simd for Avx512 {
371 type f32s = f32x16<Self>;
372 type f64s = f64x8<Self>;
373 type u8s = u8x64<Self>;
374 type i8s = i8x64<Self>;
375 type u16s = u16x32<Self>;
376 type i16s = i16x32<Self>;
377 type u32s = u32x16<Self>;
378 type i32s = i32x16<Self>;
379 type u64s = u64x8<Self>;
380 type i64s = i64x8<Self>;
381 type mask8s = mask8x64<Self>;
382 type mask16s = mask16x32<Self>;
383 type mask32s = mask32x16<Self>;
384 type mask64s = mask64x8<Self>;
385 #[inline(always)]
386 fn level(self) -> Level {
387 Level::Avx512(self)
388 }
389 #[inline]
390 fn vectorize<F: FnOnce() -> R, R>(self, f: F) -> R {
391 #[inline]
392 #[target_feature(
393 enable = "adx,aes,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512f,avx512ifma,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vpopcntdq,bmi1,bmi2,cmpxchg16b,fma,fxsr,gfni,lzcnt,movbe,pclmulqdq,popcnt,rdrand,rdseed,sha,vaes,vpclmulqdq,xsave,xsavec,xsaveopt,xsaves"
394 )]
395 fn vectorize_avx512<F: FnOnce() -> R, R>(f: F) -> R {
396 f()
397 }
398 unsafe { vectorize_avx512(f) }
399 }
400 #[inline(always)]
401 fn abs_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
402 crate::kernel!(
403 #[inline(always)]
404 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
405 _mm_andnot_ps(_mm_set1_ps(-0.0), a.into()).simd_into(token)
406 }
407 );
408 kernel(self, a)
409 }
410 #[inline(always)]
411 fn splat_f32x4(self, val: f32) -> f32x4<Self> {
412 crate::kernel!(
413 #[inline(always)]
414 fn kernel(token: Avx512, val: f32) -> f32x4<Avx512> {
415 _mm_set1_ps(val).simd_into(token)
416 }
417 );
418 kernel(self, val)
419 }
420 #[inline(always)]
421 fn reverse_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
422 let indices: u8x16<Self> =
423 [12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3].simd_into(self);
424 self.swizzle_dyn_f32x4(a, indices)
425 }
426 #[inline(always)]
427 fn slide_f32x4<const SHIFT: usize>(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
428 if SHIFT >= 4usize {
429 return b;
430 }
431 let result = dyn_alignr_128(
432 self,
433 Bytes::to_bytes(b).val.0,
434 Bytes::to_bytes(a).val.0,
435 SHIFT * 4usize,
436 );
437 Bytes::from_bytes(u8x16 {
438 val: crate::support::Aligned128(result),
439 simd: self,
440 })
441 }
442 #[inline(always)]
443 fn compress_f32x4(self, values: f32x4<Self>, mask: mask32x4<Self>) -> f32x4<Self> {
444 crate::kernel!(
445 #[inline(always)]
446 fn kernel(
447 token: Avx512,
448 values: f32x4<Avx512>,
449 mask: mask32x4<Avx512>,
450 ) -> f32x4<Avx512> {
451 _mm_maskz_compress_ps(u64::from((mask).val) as u8, values.into()).simd_into(token)
452 }
453 );
454 kernel(self, values, mask)
455 }
456 #[inline(always)]
457 fn compress_merge_f32x4(
458 self,
459 values: f32x4<Self>,
460 mask: mask32x4<Self>,
461 merge: f32x4<Self>,
462 ) -> f32x4<Self> {
463 crate::kernel!(
464 #[inline(always)]
465 fn kernel(
466 token: Avx512,
467 values: f32x4<Avx512>,
468 mask: mask32x4<Avx512>,
469 merge: f32x4<Avx512>,
470 ) -> f32x4<Avx512> {
471 _mm_mask_compress_ps(merge.into(), u64::from((mask).val) as u8, values.into())
472 .simd_into(token)
473 }
474 );
475 kernel(self, values, mask, merge)
476 }
477 #[inline(always)]
478 fn expand_f32x4(self, values: f32x4<Self>, mask: mask32x4<Self>) -> f32x4<Self> {
479 crate::kernel!(
480 #[inline(always)]
481 fn kernel(
482 token: Avx512,
483 values: f32x4<Avx512>,
484 mask: mask32x4<Avx512>,
485 ) -> f32x4<Avx512> {
486 _mm_maskz_expand_ps(u64::from((mask).val) as u8, values.into()).simd_into(token)
487 }
488 );
489 kernel(self, values, mask)
490 }
491 #[inline(always)]
492 fn expand_merge_f32x4(
493 self,
494 values: f32x4<Self>,
495 mask: mask32x4<Self>,
496 merge: f32x4<Self>,
497 ) -> f32x4<Self> {
498 crate::kernel!(
499 #[inline(always)]
500 fn kernel(
501 token: Avx512,
502 values: f32x4<Avx512>,
503 mask: mask32x4<Avx512>,
504 merge: f32x4<Avx512>,
505 ) -> f32x4<Avx512> {
506 _mm_mask_expand_ps(merge.into(), u64::from((mask).val) as u8, values.into())
507 .simd_into(token)
508 }
509 );
510 kernel(self, values, mask, merge)
511 }
512 #[inline(always)]
513 fn neg_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
514 crate::kernel!(
515 #[inline(always)]
516 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
517 _mm_xor_ps(a.into(), _mm_set1_ps(-0.0)).simd_into(token)
518 }
519 );
520 kernel(self, a)
521 }
522 #[inline(always)]
523 fn sqrt_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
524 crate::kernel!(
525 #[inline(always)]
526 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
527 _mm_sqrt_ps(a.into()).simd_into(token)
528 }
529 );
530 kernel(self, a)
531 }
532 #[inline(always)]
533 fn approximate_recip_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
534 crate::kernel!(
535 #[inline(always)]
536 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
537 _mm_rcp14_ps(a.into()).simd_into(token)
538 }
539 );
540 kernel(self, a)
541 }
542 #[inline(always)]
543 fn add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
544 crate::kernel!(
545 #[inline(always)]
546 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
547 _mm_add_ps(a.into(), b.into()).simd_into(token)
548 }
549 );
550 kernel(self, a, b)
551 }
552 #[inline(always)]
553 fn sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
554 crate::kernel!(
555 #[inline(always)]
556 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
557 _mm_sub_ps(a.into(), b.into()).simd_into(token)
558 }
559 );
560 kernel(self, a, b)
561 }
562 #[inline(always)]
563 fn mul_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
564 crate::kernel!(
565 #[inline(always)]
566 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
567 _mm_mul_ps(a.into(), b.into()).simd_into(token)
568 }
569 );
570 kernel(self, a, b)
571 }
572 #[inline(always)]
573 fn div_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
574 crate::kernel!(
575 #[inline(always)]
576 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
577 _mm_div_ps(a.into(), b.into()).simd_into(token)
578 }
579 );
580 kernel(self, a, b)
581 }
582 #[inline(always)]
583 fn copysign_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
584 crate::kernel!(
585 #[inline(always)]
586 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
587 let mask = _mm_set1_ps(-0.0);
588 _mm_or_ps(_mm_and_ps(mask, b.into()), _mm_andnot_ps(mask, a.into()))
589 .simd_into(token)
590 }
591 );
592 kernel(self, a, b)
593 }
594 #[inline(always)]
595 fn reduce_max_f32x4(self, a: f32x4<Self>) -> f32 {
596 crate::kernel!(
597 #[inline(always)]
598 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32 {
599 let reduced: __m128 = a.into();
600 let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
601 let reduced = { _mm_max_ps(reduced, shifted) };
602 let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
603 let reduced = { _mm_max_ps(reduced, shifted) };
604 _mm_cvtss_f32(reduced)
605 }
606 );
607 kernel(self, a)
608 }
609 #[inline(always)]
610 fn reduce_min_f32x4(self, a: f32x4<Self>) -> f32 {
611 crate::kernel!(
612 #[inline(always)]
613 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32 {
614 let reduced: __m128 = a.into();
615 let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
616 let reduced = { _mm_min_ps(reduced, shifted) };
617 let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
618 let reduced = { _mm_min_ps(reduced, shifted) };
619 _mm_cvtss_f32(reduced)
620 }
621 );
622 kernel(self, a)
623 }
624 #[inline(always)]
625 fn reduce_max_precise_f32x4(self, a: f32x4<Self>) -> f32 {
626 crate::kernel!(
627 #[inline(always)]
628 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32 {
629 let reduced: __m128 = a.into();
630 let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
631 let reduced = { _mm_range_ps::<5i32>(reduced, shifted) };
632 let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
633 let reduced = { _mm_range_ps::<5i32>(reduced, shifted) };
634 _mm_cvtss_f32(reduced)
635 }
636 );
637 kernel(self, a)
638 }
639 #[inline(always)]
640 fn reduce_min_precise_f32x4(self, a: f32x4<Self>) -> f32 {
641 crate::kernel!(
642 #[inline(always)]
643 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32 {
644 let reduced: __m128 = a.into();
645 let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
646 let reduced = { _mm_range_ps::<4i32>(reduced, shifted) };
647 let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
648 let reduced = { _mm_range_ps::<4i32>(reduced, shifted) };
649 _mm_cvtss_f32(reduced)
650 }
651 );
652 kernel(self, a)
653 }
654 #[inline(always)]
655 fn reduce_sum_f32x4(self, a: f32x4<Self>) -> f32 {
656 crate::kernel!(
657 #[inline(always)]
658 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32 {
659 let a: __m128 = a.into();
660 let adjacent = _mm_add_ps(a, _mm_shuffle_ps::<0b10_11_00_01>(a, a));
661 _mm_cvtss_f32(_mm_add_ss(adjacent, _mm_movehl_ps(adjacent, adjacent)))
662 }
663 );
664 kernel(self, a)
665 }
666 #[inline(always)]
667 fn reduce_product_f32x4(self, a: f32x4<Self>) -> f32 {
668 crate::kernel!(
669 #[inline(always)]
670 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32 {
671 let a: __m128 = a.into();
672 let adjacent = _mm_mul_ps(a, _mm_shuffle_ps::<0b10_11_00_01>(a, a));
673 _mm_cvtss_f32(_mm_mul_ss(adjacent, _mm_movehl_ps(adjacent, adjacent)))
674 }
675 );
676 kernel(self, a)
677 }
678 #[inline(always)]
679 fn max_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
680 crate::kernel!(
681 #[inline(always)]
682 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
683 _mm_max_ps(a.into(), b.into()).simd_into(token)
684 }
685 );
686 kernel(self, a, b)
687 }
688 #[inline(always)]
689 fn min_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
690 crate::kernel!(
691 #[inline(always)]
692 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
693 _mm_min_ps(a.into(), b.into()).simd_into(token)
694 }
695 );
696 kernel(self, a, b)
697 }
698 #[inline(always)]
699 fn max_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
700 crate::kernel!(
701 #[inline(always)]
702 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
703 _mm_range_ps::<5i32>(a.into(), b.into()).simd_into(token)
704 }
705 );
706 kernel(self, a, b)
707 }
708 #[inline(always)]
709 fn min_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
710 crate::kernel!(
711 #[inline(always)]
712 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
713 _mm_range_ps::<4i32>(a.into(), b.into()).simd_into(token)
714 }
715 );
716 kernel(self, a, b)
717 }
718 #[inline(always)]
719 fn clamp_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
720 crate::kernel!(
721 #[inline(always)]
722 fn kernel(
723 token: Avx512,
724 a: f32x4<Avx512>,
725 b: f32x4<Avx512>,
726 c: f32x4<Avx512>,
727 ) -> f32x4<Avx512> {
728 debug_assert!(
729 b.simd_le(c).all_true(),
730 "clamp requires min <= max for all elements"
731 );
732 _mm_min_ps(c.into(), _mm_max_ps(b.into(), a.into())).simd_into(token)
733 }
734 );
735 kernel(self, a, b, c)
736 }
737 #[inline(always)]
738 fn simd_eq_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
739 crate::kernel!(
740 #[inline(always)]
741 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> mask32x4<Avx512> {
742 mask32x4 {
743 val: _mm_cmp_ps_mask::<0i32>(a.into(), b.into()),
744 simd: token,
745 }
746 }
747 );
748 kernel(self, a, b)
749 }
750 #[inline(always)]
751 fn simd_ne_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
752 crate::kernel!(
753 #[inline(always)]
754 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> mask32x4<Avx512> {
755 mask32x4 {
756 val: _mm_cmp_ps_mask::<4i32>(a.into(), b.into()),
757 simd: token,
758 }
759 }
760 );
761 kernel(self, a, b)
762 }
763 #[inline(always)]
764 fn simd_lt_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
765 crate::kernel!(
766 #[inline(always)]
767 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> mask32x4<Avx512> {
768 mask32x4 {
769 val: _mm_cmp_ps_mask::<17i32>(a.into(), b.into()),
770 simd: token,
771 }
772 }
773 );
774 kernel(self, a, b)
775 }
776 #[inline(always)]
777 fn simd_le_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
778 crate::kernel!(
779 #[inline(always)]
780 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> mask32x4<Avx512> {
781 mask32x4 {
782 val: _mm_cmp_ps_mask::<18i32>(a.into(), b.into()),
783 simd: token,
784 }
785 }
786 );
787 kernel(self, a, b)
788 }
789 #[inline(always)]
790 fn zip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
791 crate::kernel!(
792 #[inline(always)]
793 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
794 _mm_unpacklo_ps(a.into(), b.into()).simd_into(token)
795 }
796 );
797 kernel(self, a, b)
798 }
799 #[inline(always)]
800 fn zip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
801 crate::kernel!(
802 #[inline(always)]
803 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
804 _mm_unpackhi_ps(a.into(), b.into()).simd_into(token)
805 }
806 );
807 kernel(self, a, b)
808 }
809 #[inline(always)]
810 fn unzip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
811 crate::kernel!(
812 #[inline(always)]
813 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
814 _mm_shuffle_ps::<0b10_00_10_00>(a.into(), b.into()).simd_into(token)
815 }
816 );
817 kernel(self, a, b)
818 }
819 #[inline(always)]
820 fn unzip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
821 crate::kernel!(
822 #[inline(always)]
823 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
824 _mm_shuffle_ps::<0b11_01_11_01>(a.into(), b.into()).simd_into(token)
825 }
826 );
827 kernel(self, a, b)
828 }
829 #[inline(always)]
830 fn interleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
831 (self.zip_low_f32x4(a, b), self.zip_high_f32x4(a, b))
832 }
833 #[inline(always)]
834 fn deinterleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
835 (self.unzip_low_f32x4(a, b), self.unzip_high_f32x4(a, b))
836 }
837 #[inline(always)]
838 fn mul_add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
839 crate::kernel!(
840 #[inline(always)]
841 fn kernel(
842 token: Avx512,
843 a: f32x4<Avx512>,
844 b: f32x4<Avx512>,
845 c: f32x4<Avx512>,
846 ) -> f32x4<Avx512> {
847 _mm_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token)
848 }
849 );
850 kernel(self, a, b, c)
851 }
852 #[inline(always)]
853 fn mul_add_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
854 self.mul_add_f32x4(a, b, c)
855 }
856 #[inline(always)]
857 fn mul_sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
858 crate::kernel!(
859 #[inline(always)]
860 fn kernel(
861 token: Avx512,
862 a: f32x4<Avx512>,
863 b: f32x4<Avx512>,
864 c: f32x4<Avx512>,
865 ) -> f32x4<Avx512> {
866 _mm_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token)
867 }
868 );
869 kernel(self, a, b, c)
870 }
871 #[inline(always)]
872 fn mul_sub_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
873 self.mul_sub_f32x4(a, b, c)
874 }
875 #[inline(always)]
876 fn floor_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
877 crate::kernel!(
878 #[inline(always)]
879 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
880 _mm_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
881 .simd_into(token)
882 }
883 );
884 kernel(self, a)
885 }
886 #[inline(always)]
887 fn ceil_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
888 crate::kernel!(
889 #[inline(always)]
890 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
891 _mm_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
892 .simd_into(token)
893 }
894 );
895 kernel(self, a)
896 }
897 #[inline(always)]
898 fn round_ties_even_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
899 crate::kernel!(
900 #[inline(always)]
901 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
902 _mm_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
903 .simd_into(token)
904 }
905 );
906 kernel(self, a)
907 }
908 #[inline(always)]
909 fn round_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
910 let bias = self.splat_f32x4(const { f32::next_down(0.5) });
911 (a + self.copysign_f32x4(bias, a)).trunc()
912 }
913 #[inline(always)]
914 fn fract_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
915 a - self.trunc_f32x4(a)
916 }
917 #[inline(always)]
918 fn trunc_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
919 crate::kernel!(
920 #[inline(always)]
921 fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
922 _mm_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
923 .simd_into(token)
924 }
925 );
926 kernel(self, a)
927 }
928 #[inline(always)]
929 fn to_degrees_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
930 a * const { f32::to_degrees(1.0) }
931 }
932 #[inline(always)]
933 fn to_radians_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
934 a * const { f32::to_radians(1.0) }
935 }
936 #[inline(always)]
937 fn is_nan_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
938 crate::kernel!(
939 #[inline(always)]
940 fn kernel(token: Avx512, a: f32x4<Avx512>) -> mask32x4<Avx512> {
941 mask32x4 {
942 val: _mm_fpclass_ps_mask::<129i32>(a.into()),
943 simd: token,
944 }
945 }
946 );
947 kernel(self, a)
948 }
949 #[inline(always)]
950 fn is_infinite_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
951 crate::kernel!(
952 #[inline(always)]
953 fn kernel(token: Avx512, a: f32x4<Avx512>) -> mask32x4<Avx512> {
954 mask32x4 {
955 val: _mm_fpclass_ps_mask::<24i32>(a.into()),
956 simd: token,
957 }
958 }
959 );
960 kernel(self, a)
961 }
962 #[inline(always)]
963 fn is_finite_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
964 crate::kernel!(
965 #[inline(always)]
966 fn kernel(token: Avx512, a: f32x4<Avx512>) -> mask32x4<Avx512> {
967 !(mask32x4 {
968 val: _mm_fpclass_ps_mask::<153i32>(a.into()),
969 simd: token,
970 })
971 }
972 );
973 kernel(self, a)
974 }
975 #[inline(always)]
976 fn is_subnormal_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
977 crate::kernel!(
978 #[inline(always)]
979 fn kernel(token: Avx512, a: f32x4<Avx512>) -> mask32x4<Avx512> {
980 mask32x4 {
981 val: _mm_fpclass_ps_mask::<129i32>(_mm_xor_ps(
982 a.into(),
983 _mm_set1_ps(f32::INFINITY),
984 )),
985 simd: token,
986 }
987 }
988 );
989 kernel(self, a)
990 }
991 #[inline(always)]
992 fn is_normal_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
993 crate::kernel!(
994 #[inline(always)]
995 fn kernel(token: Avx512, a: f32x4<Avx512>) -> mask32x4<Avx512> {
996 !(mask32x4 {
997 val: _mm_fpclass_ps_mask::<191i32>(a.into()),
998 simd: token,
999 })
1000 }
1001 );
1002 kernel(self, a)
1003 }
1004 #[inline(always)]
1005 fn is_sign_positive_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
1006 let i: i32x4<Self> = a.bitcast();
1007 (i & i32::MIN).simd_eq(0)
1008 }
1009 #[inline(always)]
1010 fn is_sign_negative_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
1011 !a.is_sign_positive()
1012 }
1013 #[inline(always)]
1014 fn select_f32x4(self, a: mask32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
1015 crate::kernel!(
1016 #[inline(always)]
1017 fn kernel(
1018 token: Avx512,
1019 a: mask32x4<Avx512>,
1020 b: f32x4<Avx512>,
1021 c: f32x4<Avx512>,
1022 ) -> f32x4<Avx512> {
1023 _mm_mask_blend_ps(a.val, c.into(), b.into()).simd_into(token)
1024 }
1025 );
1026 kernel(self, a, b, c)
1027 }
1028 #[inline(always)]
1029 fn combine_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x8<Self> {
1030 crate::kernel!(
1031 #[inline(always)]
1032 fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x8<Avx512> {
1033 _mm256_setr_m128(a.into(), b.into()).simd_into(token)
1034 }
1035 );
1036 kernel(self, a, b)
1037 }
1038 #[inline(always)]
1039 fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4<Self>; 4usize] {
1040 crate::kernel!(
1041 #[inline(always)]
1042 fn kernel(token: Avx512, src: &[f32; 16usize]) -> [f32x4<Avx512>; 4usize] {
1043 let lanes: __m512 =
1044 crate::transmute::checked_transmute_copy::<[f32; 16usize], __m512>(src);
1045 let lanes = _mm512_permutexvar_ps(
1046 _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15),
1047 lanes,
1048 );
1049 let outputs: [__m128; 4] = crate::transmute::checked_transmute_copy(&lanes);
1050 [
1051 outputs[0].simd_into(token),
1052 outputs[1].simd_into(token),
1053 outputs[2].simd_into(token),
1054 outputs[3].simd_into(token),
1055 ]
1056 }
1057 );
1058 kernel(self, src)
1059 }
1060 #[inline(always)]
1061 fn store_four_interleaved_f32x4(
1062 self,
1063 vectors: [f32x4<Self>; 4usize],
1064 dest: &mut [f32; 16usize],
1065 ) -> () {
1066 crate::kernel!(
1067 #[inline(always)]
1068 fn kernel(
1069 token: Avx512,
1070 vectors: [f32x4<Avx512>; 4usize],
1071 dest: &mut [f32; 16usize],
1072 ) -> () {
1073 let _ = token;
1074 let inputs: [__m128; 4] = [
1075 vectors[0].into(),
1076 vectors[1].into(),
1077 vectors[2].into(),
1078 vectors[3].into(),
1079 ];
1080 let lanes: __m512 = crate::transmute::checked_transmute_copy(&inputs);
1081 let lanes = _mm512_permutexvar_ps(
1082 _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15),
1083 lanes,
1084 );
1085 crate::transmute::checked_transmute_store::<__m512, [f32; 16usize]>(lanes, dest);
1086 }
1087 );
1088 kernel(self, vectors, dest);
1089 }
1090 #[inline(always)]
1091 fn widen_f32x4(self, a: f32x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
1092 crate::kernel!(
1093 #[inline(always)]
1094 fn kernel(token: Avx512, a: f32x4<Avx512>) -> (f64x2<Avx512>, f64x2<Avx512>) {
1095 let raw = a.into();
1096 (
1097 _mm_cvtps_pd(raw).simd_into(token),
1098 _mm_cvtps_pd(_mm_movehl_ps(raw, raw)).simd_into(token),
1099 )
1100 }
1101 );
1102 kernel(self, a)
1103 }
1104 #[inline(always)]
1105 fn cvt_u32_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
1106 crate::kernel!(
1107 #[inline(always)]
1108 fn kernel(token: Avx512, a: f32x4<Avx512>) -> u32x4<Avx512> {
1109 _mm_cvttps_epu32(a.into()).simd_into(token)
1110 }
1111 );
1112 kernel(self, a)
1113 }
1114 #[inline(always)]
1115 fn cvt_u32_precise_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
1116 crate::kernel!(
1117 #[inline(always)]
1118 fn kernel(token: Avx512, a: f32x4<Avx512>) -> u32x4<Avx512> {
1119 let a = a.into();
1120 let positive = _mm_cmp_ps_mask::<17i32>(_mm_setzero_ps(), a);
1121 _mm_maskz_cvttps_epu32(positive, a).simd_into(token)
1122 }
1123 );
1124 kernel(self, a)
1125 }
1126 #[inline(always)]
1127 fn cvt_i32_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
1128 crate::kernel!(
1129 #[inline(always)]
1130 fn kernel(token: Avx512, a: f32x4<Avx512>) -> i32x4<Avx512> {
1131 _mm_cvttps_epi32(a.into()).simd_into(token)
1132 }
1133 );
1134 kernel(self, a)
1135 }
1136 #[inline(always)]
1137 fn cvt_i32_precise_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
1138 crate::kernel!(
1139 #[inline(always)]
1140 fn kernel(token: Avx512, a: f32x4<Avx512>) -> i32x4<Avx512> {
1141 let a = a.into();
1142 let in_range = _mm_cmp_ps_mask::<17i32>(a, _mm_set1_ps(2147483648.0));
1143 let mut converted = _mm_mask_cvttps_epi32(_mm_set1_epi32(i32::MAX), in_range, a);
1144 let is_not_nan = _mm_cmp_ps_mask::<7i32>(a, a);
1145 converted = _mm_mask_blend_epi32(is_not_nan, _mm_setzero_si128(), converted);
1146 converted.simd_into(token)
1147 }
1148 );
1149 kernel(self, a)
1150 }
1151 #[inline(always)]
1152 fn abs_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
1153 crate::kernel!(
1154 #[inline(always)]
1155 fn kernel(token: Avx512, a: i8x16<Avx512>) -> i8x16<Avx512> {
1156 _mm_abs_epi8(a.into()).simd_into(token)
1157 }
1158 );
1159 kernel(self, a)
1160 }
1161 #[inline(always)]
1162 fn splat_i8x16(self, val: i8) -> i8x16<Self> {
1163 crate::kernel!(
1164 #[inline(always)]
1165 fn kernel(token: Avx512, val: i8) -> i8x16<Avx512> {
1166 _mm_set1_epi8(val).simd_into(token)
1167 }
1168 );
1169 kernel(self, val)
1170 }
1171 #[inline(always)]
1172 fn reverse_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
1173 let indices: u8x16<Self> =
1174 [15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0].simd_into(self);
1175 self.swizzle_dyn_i8x16(a, indices)
1176 }
1177 #[inline(always)]
1178 fn slide_i8x16<const SHIFT: usize>(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1179 if SHIFT >= 16usize {
1180 return b;
1181 }
1182 let result = dyn_alignr_128(
1183 self,
1184 Bytes::to_bytes(b).val.0,
1185 Bytes::to_bytes(a).val.0,
1186 SHIFT,
1187 );
1188 Bytes::from_bytes(u8x16 {
1189 val: crate::support::Aligned128(result),
1190 simd: self,
1191 })
1192 }
1193 #[inline(always)]
1194 fn compress_i8x16(self, values: i8x16<Self>, mask: mask8x16<Self>) -> i8x16<Self> {
1195 crate::kernel!(
1196 #[inline(always)]
1197 fn kernel(
1198 token: Avx512,
1199 values: i8x16<Avx512>,
1200 mask: mask8x16<Avx512>,
1201 ) -> i8x16<Avx512> {
1202 _mm_maskz_compress_epi8(u64::from((mask).val) as u16, values.into())
1203 .simd_into(token)
1204 }
1205 );
1206 kernel(self, values, mask)
1207 }
1208 #[inline(always)]
1209 fn compress_merge_i8x16(
1210 self,
1211 values: i8x16<Self>,
1212 mask: mask8x16<Self>,
1213 merge: i8x16<Self>,
1214 ) -> i8x16<Self> {
1215 crate::kernel!(
1216 #[inline(always)]
1217 fn kernel(
1218 token: Avx512,
1219 values: i8x16<Avx512>,
1220 mask: mask8x16<Avx512>,
1221 merge: i8x16<Avx512>,
1222 ) -> i8x16<Avx512> {
1223 _mm_mask_compress_epi8(merge.into(), u64::from((mask).val) as u16, values.into())
1224 .simd_into(token)
1225 }
1226 );
1227 kernel(self, values, mask, merge)
1228 }
1229 #[inline(always)]
1230 fn expand_i8x16(self, values: i8x16<Self>, mask: mask8x16<Self>) -> i8x16<Self> {
1231 crate::kernel!(
1232 #[inline(always)]
1233 fn kernel(
1234 token: Avx512,
1235 values: i8x16<Avx512>,
1236 mask: mask8x16<Avx512>,
1237 ) -> i8x16<Avx512> {
1238 _mm_maskz_expand_epi8(u64::from((mask).val) as u16, values.into()).simd_into(token)
1239 }
1240 );
1241 kernel(self, values, mask)
1242 }
1243 #[inline(always)]
1244 fn expand_merge_i8x16(
1245 self,
1246 values: i8x16<Self>,
1247 mask: mask8x16<Self>,
1248 merge: i8x16<Self>,
1249 ) -> i8x16<Self> {
1250 crate::kernel!(
1251 #[inline(always)]
1252 fn kernel(
1253 token: Avx512,
1254 values: i8x16<Avx512>,
1255 mask: mask8x16<Avx512>,
1256 merge: i8x16<Avx512>,
1257 ) -> i8x16<Avx512> {
1258 _mm_mask_expand_epi8(merge.into(), u64::from((mask).val) as u16, values.into())
1259 .simd_into(token)
1260 }
1261 );
1262 kernel(self, values, mask, merge)
1263 }
1264 #[inline(always)]
1265 fn count_ones_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
1266 crate::kernel!(
1267 #[inline(always)]
1268 fn kernel(token: Avx512, a: i8x16<Avx512>) -> i8x16<Avx512> {
1269 _mm_popcnt_epi8(a.into()).simd_into(token)
1270 }
1271 );
1272 kernel(self, a)
1273 }
1274 #[inline(always)]
1275 fn count_zeros_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
1276 self.count_ones_i8x16(self.not_i8x16(a))
1277 }
1278 #[inline(always)]
1279 fn add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1280 crate::kernel!(
1281 #[inline(always)]
1282 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1283 _mm_add_epi8(a.into(), b.into()).simd_into(token)
1284 }
1285 );
1286 kernel(self, a, b)
1287 }
1288 #[inline(always)]
1289 fn saturating_add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1290 crate::kernel!(
1291 #[inline(always)]
1292 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1293 _mm_adds_epi8(a.into(), b.into()).simd_into(token)
1294 }
1295 );
1296 kernel(self, a, b)
1297 }
1298 #[inline(always)]
1299 fn sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1300 crate::kernel!(
1301 #[inline(always)]
1302 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1303 _mm_sub_epi8(a.into(), b.into()).simd_into(token)
1304 }
1305 );
1306 kernel(self, a, b)
1307 }
1308 #[inline(always)]
1309 fn saturating_sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1310 crate::kernel!(
1311 #[inline(always)]
1312 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1313 _mm_subs_epi8(a.into(), b.into()).simd_into(token)
1314 }
1315 );
1316 kernel(self, a, b)
1317 }
1318 #[inline(always)]
1319 fn mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1320 crate::kernel!(
1321 #[inline(always)]
1322 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1323 let a = a.into();
1324 let b = b.into();
1325 let low_mask = _mm_set1_epi16(0xFF);
1326 let dst_even = _mm_mullo_epi16(a, b);
1327 let dst_odd = _mm_maddubs_epi16(a, _mm_andnot_si128(low_mask, b));
1328 _mm_or_si128(
1329 _mm_slli_epi16(dst_odd, 8),
1330 _mm_and_si128(dst_even, low_mask),
1331 )
1332 .simd_into(token)
1333 }
1334 );
1335 kernel(self, a, b)
1336 }
1337 #[inline(always)]
1338 fn and_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1339 crate::kernel!(
1340 #[inline(always)]
1341 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1342 _mm_and_si128(a.into(), b.into()).simd_into(token)
1343 }
1344 );
1345 kernel(self, a, b)
1346 }
1347 #[inline(always)]
1348 fn or_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1349 crate::kernel!(
1350 #[inline(always)]
1351 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1352 _mm_or_si128(a.into(), b.into()).simd_into(token)
1353 }
1354 );
1355 kernel(self, a, b)
1356 }
1357 #[inline(always)]
1358 fn xor_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1359 crate::kernel!(
1360 #[inline(always)]
1361 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1362 _mm_xor_si128(a.into(), b.into()).simd_into(token)
1363 }
1364 );
1365 kernel(self, a, b)
1366 }
1367 #[inline(always)]
1368 fn not_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
1369 a ^ !0
1370 }
1371 #[inline(always)]
1372 fn shl_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
1373 crate::kernel!(
1374 #[inline(always)]
1375 fn kernel(token: Avx512, a: i8x16<Avx512>, shift: u32) -> i8x16<Avx512> {
1376 let val = a.into();
1377 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1378 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
1379 let byte_mask = _mm_set1_epi8(mask_byte);
1380 _mm_sll_epi16(_mm_and_si128(val, byte_mask), shift_count).simd_into(token)
1381 }
1382 );
1383 kernel(self, a, shift)
1384 }
1385 #[inline(always)]
1386 fn shlv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1387 crate::kernel!(
1388 #[inline(always)]
1389 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1390 let val = a.into();
1391 let counts = b.into();
1392 let byte_mask = _mm_set1_epi16(0x00ff);
1393 let lo_counts = _mm_and_si128(counts, byte_mask);
1394 let hi_counts = _mm_srli_epi16::<8>(counts);
1395 let lo_shifted = _mm_sllv_epi16(val, lo_counts);
1396 let hi_values = _mm_andnot_si128(byte_mask, val);
1397 let hi_shifted = _mm_sllv_epi16(hi_values, hi_counts);
1398 _mm_mask_blend_epi8(0xaaaa_u16, lo_shifted, hi_shifted).simd_into(token)
1399 }
1400 );
1401 kernel(self, a, b)
1402 }
1403 #[inline(always)]
1404 fn shr_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
1405 crate::kernel!(
1406 #[inline(always)]
1407 fn kernel(token: Avx512, a: i8x16<Avx512>, shift: u32) -> i8x16<Avx512> {
1408 let val = a.into();
1409 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1410 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
1411 let byte_mask = _mm_set1_epi8(mask_byte);
1412 let shifted = _mm_srl_epi16(val, shift_count);
1413 let result = {
1414 let sign = _mm_cmpgt_epi8(_mm_setzero_si128(), val);
1415 _mm_ternarylogic_epi32::<0xca>(byte_mask, shifted, sign)
1416 };
1417 result.simd_into(token)
1418 }
1419 );
1420 kernel(self, a, shift)
1421 }
1422 #[inline(always)]
1423 fn shrv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1424 crate::kernel!(
1425 #[inline(always)]
1426 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1427 let val = a.into();
1428 let counts = b.into();
1429 let zero = _mm_setzero_si128();
1430 let value_extend = _mm_cmpgt_epi8(zero, val);
1431 let lo_values = _mm_unpacklo_epi8(val, value_extend);
1432 let hi_values = _mm_unpackhi_epi8(val, value_extend);
1433 let lo_counts = _mm_unpacklo_epi8(counts, zero);
1434 let hi_counts = _mm_unpackhi_epi8(counts, zero);
1435 let byte_mask = _mm_set1_epi16(0x00ff);
1436 let lo_shifted = _mm_and_si128(_mm_srav_epi16(lo_values, lo_counts), byte_mask);
1437 let hi_shifted = _mm_and_si128(_mm_srav_epi16(hi_values, hi_counts), byte_mask);
1438 _mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
1439 }
1440 );
1441 kernel(self, a, b)
1442 }
1443 #[inline(always)]
1444 fn reduce_max_i8x16(self, a: i8x16<Self>) -> i8 {
1445 crate::kernel!(
1446 #[inline(always)]
1447 fn kernel(token: Avx512, a: i8x16<Avx512>) -> i8 {
1448 let reduced: __m128i = a.into();
1449 let shifted = _mm_srli_si128::<8>(reduced);
1450 let reduced = { _mm_max_epi8(reduced, shifted) };
1451 let shifted = _mm_srli_si128::<4>(reduced);
1452 let reduced = { _mm_max_epi8(reduced, shifted) };
1453 let shifted = _mm_srli_si128::<2>(reduced);
1454 let reduced = { _mm_max_epi8(reduced, shifted) };
1455 let shifted = _mm_srli_si128::<1>(reduced);
1456 let reduced = { _mm_max_epi8(reduced, shifted) };
1457 {
1458 let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
1459 lanes[0]
1460 }
1461 }
1462 );
1463 kernel(self, a)
1464 }
1465 #[inline(always)]
1466 fn reduce_min_i8x16(self, a: i8x16<Self>) -> i8 {
1467 crate::kernel!(
1468 #[inline(always)]
1469 fn kernel(token: Avx512, a: i8x16<Avx512>) -> i8 {
1470 let reduced: __m128i = a.into();
1471 let shifted = _mm_srli_si128::<8>(reduced);
1472 let reduced = { _mm_min_epi8(reduced, shifted) };
1473 let shifted = _mm_srli_si128::<4>(reduced);
1474 let reduced = { _mm_min_epi8(reduced, shifted) };
1475 let shifted = _mm_srli_si128::<2>(reduced);
1476 let reduced = { _mm_min_epi8(reduced, shifted) };
1477 let shifted = _mm_srli_si128::<1>(reduced);
1478 let reduced = { _mm_min_epi8(reduced, shifted) };
1479 {
1480 let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
1481 lanes[0]
1482 }
1483 }
1484 );
1485 kernel(self, a)
1486 }
1487 #[inline(always)]
1488 fn reduce_sum_i8x16(self, a: i8x16<Self>) -> i8 {
1489 crate::kernel!(
1490 #[inline(always)]
1491 fn kernel(token: Avx512, a: i8x16<Avx512>) -> i8 {
1492 let sum: __m128i = a.into();
1493 let sum = _mm_add_epi8(sum, _mm_srli_si128::<8>(sum));
1494 let sum = _mm_add_epi8(sum, _mm_srli_si128::<4>(sum));
1495 let sum = _mm_add_epi8(sum, _mm_srli_si128::<2>(sum));
1496 let sum = _mm_add_epi8(sum, _mm_srli_si128::<1>(sum));
1497 let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&sum);
1498 lanes[0]
1499 }
1500 );
1501 kernel(self, a)
1502 }
1503 #[inline(always)]
1504 fn reduce_product_i8x16(self, a: i8x16<Self>) -> i8 {
1505 crate::kernel!(
1506 #[inline(always)]
1507 fn kernel(token: Avx512, a: i8x16<Avx512>) -> i8 {
1508 let value: __m128i = a.into();
1509 let product = _mm_mullo_epi16(
1510 _mm_unpacklo_epi8(value, value),
1511 _mm_unpackhi_epi8(value, value),
1512 );
1513 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
1514 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
1515 let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
1516 let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&product);
1517 lanes[0]
1518 }
1519 );
1520 kernel(self, a)
1521 }
1522 #[inline(always)]
1523 fn max_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1524 crate::kernel!(
1525 #[inline(always)]
1526 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1527 _mm_max_epi8(a.into(), b.into()).simd_into(token)
1528 }
1529 );
1530 kernel(self, a, b)
1531 }
1532 #[inline(always)]
1533 fn min_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1534 crate::kernel!(
1535 #[inline(always)]
1536 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1537 _mm_min_epi8(a.into(), b.into()).simd_into(token)
1538 }
1539 );
1540 kernel(self, a, b)
1541 }
1542 #[inline(always)]
1543 fn clamp_i8x16(self, a: i8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
1544 debug_assert!(
1545 b.simd_le(c).all_true(),
1546 "clamp requires min <= max for all elements"
1547 );
1548 a.max(b).min(c)
1549 }
1550 #[inline(always)]
1551 fn simd_eq_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1552 crate::kernel!(
1553 #[inline(always)]
1554 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> mask8x16<Avx512> {
1555 mask8x16 {
1556 val: _mm_cmpeq_epi8_mask(a.into(), b.into()),
1557 simd: token,
1558 }
1559 }
1560 );
1561 kernel(self, a, b)
1562 }
1563 #[inline(always)]
1564 fn simd_ne_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1565 crate::kernel!(
1566 #[inline(always)]
1567 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> mask8x16<Avx512> {
1568 mask8x16 {
1569 val: _mm_cmpneq_epi8_mask(a.into(), b.into()),
1570 simd: token,
1571 }
1572 }
1573 );
1574 kernel(self, a, b)
1575 }
1576 #[inline(always)]
1577 fn simd_lt_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1578 crate::kernel!(
1579 #[inline(always)]
1580 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> mask8x16<Avx512> {
1581 mask8x16 {
1582 val: _mm_cmplt_epi8_mask(a.into(), b.into()),
1583 simd: token,
1584 }
1585 }
1586 );
1587 kernel(self, a, b)
1588 }
1589 #[inline(always)]
1590 fn simd_le_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1591 crate::kernel!(
1592 #[inline(always)]
1593 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> mask8x16<Avx512> {
1594 mask8x16 {
1595 val: _mm_cmple_epi8_mask(a.into(), b.into()),
1596 simd: token,
1597 }
1598 }
1599 );
1600 kernel(self, a, b)
1601 }
1602 #[inline(always)]
1603 fn zip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1604 crate::kernel!(
1605 #[inline(always)]
1606 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1607 _mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
1608 }
1609 );
1610 kernel(self, a, b)
1611 }
1612 #[inline(always)]
1613 fn zip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1614 crate::kernel!(
1615 #[inline(always)]
1616 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1617 _mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
1618 }
1619 );
1620 kernel(self, a, b)
1621 }
1622 #[inline(always)]
1623 fn unzip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1624 crate::kernel!(
1625 #[inline(always)]
1626 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1627 _mm_permutex2var_epi8(
1628 a.into(),
1629 _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
1630 b.into(),
1631 )
1632 .simd_into(token)
1633 }
1634 );
1635 kernel(self, a, b)
1636 }
1637 #[inline(always)]
1638 fn unzip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1639 crate::kernel!(
1640 #[inline(always)]
1641 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
1642 _mm_permutex2var_epi8(
1643 a.into(),
1644 _mm_setr_epi8(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
1645 b.into(),
1646 )
1647 .simd_into(token)
1648 }
1649 );
1650 kernel(self, a, b)
1651 }
1652 #[inline(always)]
1653 fn interleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
1654 (self.zip_low_i8x16(a, b), self.zip_high_i8x16(a, b))
1655 }
1656 #[inline(always)]
1657 fn deinterleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
1658 crate::kernel!(
1659 #[inline(always)]
1660 fn kernel(
1661 token: Avx512,
1662 a: i8x16<Avx512>,
1663 b: i8x16<Avx512>,
1664 ) -> (i8x16<Avx512>, i8x16<Avx512>) {
1665 let a = a.into();
1666 let b = b.into();
1667 (
1668 _mm_permutex2var_epi8(
1669 a,
1670 _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
1671 b,
1672 )
1673 .simd_into(token),
1674 _mm_permutex2var_epi8(
1675 a,
1676 _mm_setr_epi8(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
1677 b,
1678 )
1679 .simd_into(token),
1680 )
1681 }
1682 );
1683 kernel(self, a, b)
1684 }
1685 #[inline(always)]
1686 fn select_i8x16(self, a: mask8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
1687 crate::kernel!(
1688 #[inline(always)]
1689 fn kernel(
1690 token: Avx512,
1691 a: mask8x16<Avx512>,
1692 b: i8x16<Avx512>,
1693 c: i8x16<Avx512>,
1694 ) -> i8x16<Avx512> {
1695 _mm_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token)
1696 }
1697 );
1698 kernel(self, a, b, c)
1699 }
1700 #[inline(always)]
1701 fn combine_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x32<Self> {
1702 crate::kernel!(
1703 #[inline(always)]
1704 fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x32<Avx512> {
1705 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
1706 }
1707 );
1708 kernel(self, a, b)
1709 }
1710 #[inline(always)]
1711 fn neg_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
1712 crate::kernel!(
1713 #[inline(always)]
1714 fn kernel(token: Avx512, a: i8x16<Avx512>) -> i8x16<Avx512> {
1715 _mm_sub_epi8(_mm_setzero_si128(), a.into()).simd_into(token)
1716 }
1717 );
1718 kernel(self, a)
1719 }
1720 #[inline(always)]
1721 fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16<Self>; 4usize] {
1722 crate::kernel!(
1723 #[inline(always)]
1724 fn kernel(token: Avx512, src: &[i8; 64usize]) -> [i8x16<Avx512>; 4usize] {
1725 let lanes: __m512i =
1726 crate::transmute::checked_transmute_copy::<[i8; 64usize], __m512i>(src);
1727 let lanes = _mm512_permutexvar_epi8(
1728 _mm512_set_epi8(
1729 63, 59, 55, 51, 47, 43, 39, 35, 31, 27, 23, 19, 15, 11, 7, 3, 62, 58, 54,
1730 50, 46, 42, 38, 34, 30, 26, 22, 18, 14, 10, 6, 2, 61, 57, 53, 49, 45, 41,
1731 37, 33, 29, 25, 21, 17, 13, 9, 5, 1, 60, 56, 52, 48, 44, 40, 36, 32, 28,
1732 24, 20, 16, 12, 8, 4, 0,
1733 ),
1734 lanes,
1735 );
1736 let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes);
1737 [
1738 outputs[0].simd_into(token),
1739 outputs[1].simd_into(token),
1740 outputs[2].simd_into(token),
1741 outputs[3].simd_into(token),
1742 ]
1743 }
1744 );
1745 kernel(self, src)
1746 }
1747 #[inline(always)]
1748 fn store_four_interleaved_i8x16(
1749 self,
1750 vectors: [i8x16<Self>; 4usize],
1751 dest: &mut [i8; 64usize],
1752 ) -> () {
1753 crate::kernel!(
1754 #[inline(always)]
1755 fn kernel(
1756 token: Avx512,
1757 vectors: [i8x16<Avx512>; 4usize],
1758 dest: &mut [i8; 64usize],
1759 ) -> () {
1760 let _ = token;
1761 let inputs: [__m128i; 4] = [
1762 vectors[0].into(),
1763 vectors[1].into(),
1764 vectors[2].into(),
1765 vectors[3].into(),
1766 ];
1767 let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs);
1768 let lanes = _mm512_permutexvar_epi8(
1769 _mm512_set_epi8(
1770 63, 47, 31, 15, 62, 46, 30, 14, 61, 45, 29, 13, 60, 44, 28, 12, 59, 43, 27,
1771 11, 58, 42, 26, 10, 57, 41, 25, 9, 56, 40, 24, 8, 55, 39, 23, 7, 54, 38,
1772 22, 6, 53, 37, 21, 5, 52, 36, 20, 4, 51, 35, 19, 3, 50, 34, 18, 2, 49, 33,
1773 17, 1, 48, 32, 16, 0,
1774 ),
1775 lanes,
1776 );
1777 crate::transmute::checked_transmute_store::<__m512i, [i8; 64usize]>(lanes, dest);
1778 }
1779 );
1780 kernel(self, vectors, dest);
1781 }
1782 #[inline(always)]
1783 fn widen_i8x16(self, a: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1784 crate::kernel!(
1785 #[inline(always)]
1786 fn kernel(token: Avx512, a: i8x16<Avx512>) -> (i16x8<Avx512>, i16x8<Avx512>) {
1787 let raw = a.into();
1788 (
1789 _mm_cvtepi8_epi16(raw).simd_into(token),
1790 _mm_cvtepi8_epi16(_mm_srli_si128::<8>(raw)).simd_into(token),
1791 )
1792 }
1793 );
1794 kernel(self, a)
1795 }
1796 #[inline(always)]
1797 fn widening_mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1798 let (a0, a1) = self.widen_i8x16(a);
1799 let (b0, b1) = self.widen_i8x16(b);
1800 (self.mul_i16x8(a0, b0), self.mul_i16x8(a1, b1))
1801 }
1802 #[inline(always)]
1803 fn widening_add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1804 crate::kernel!(
1805 #[inline(always)]
1806 fn kernel(
1807 token: Avx512,
1808 a: i8x16<Avx512>,
1809 b: i8x16<Avx512>,
1810 ) -> (i16x8<Avx512>, i16x8<Avx512>) {
1811 let a = a.into();
1812 let b = b.into();
1813 let low = a;
1814 let high = b;
1815 let result_low = _mm_unpacklo_epi8(low, high);
1816 let result_high = _mm_unpackhi_epi8(low, high);
1817 let ones = _mm_set1_epi8(1);
1818 let result_low = _mm_maddubs_epi16(ones, result_low);
1819 let result_high = _mm_maddubs_epi16(ones, result_high);
1820 (result_low.simd_into(token), result_high.simd_into(token))
1821 }
1822 );
1823 kernel(self, a, b)
1824 }
1825 #[inline(always)]
1826 fn pairwise_widening_add_i8x16(self, a: i8x16<Self>) -> i16x8<Self> {
1827 crate::kernel!(
1828 #[inline(always)]
1829 fn kernel(token: Avx512, a: i8x16<Avx512>) -> i16x8<Avx512> {
1830 ({ _mm_maddubs_epi16(_mm_set1_epi8(1), a.into()) }).simd_into(token)
1831 }
1832 );
1833 kernel(self, a)
1834 }
1835 #[inline(always)]
1836 fn pairwise_dot_product_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i16x8<Self> {
1837 let (low, high) = self.widening_mul_i8x16(a, b);
1838 let (even, odd) = self.deinterleave_i16x8(low, high);
1839 self.add_i16x8(even, odd)
1840 }
1841 #[inline(always)]
1842 fn splat_u8x16(self, val: u8) -> u8x16<Self> {
1843 crate::kernel!(
1844 #[inline(always)]
1845 fn kernel(token: Avx512, val: u8) -> u8x16<Avx512> {
1846 _mm_set1_epi8(val.cast_signed()).simd_into(token)
1847 }
1848 );
1849 kernel(self, val)
1850 }
1851 #[inline(always)]
1852 fn reverse_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1853 let indices: u8x16<Self> =
1854 [15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0].simd_into(self);
1855 self.swizzle_dyn_u8x16(a, indices)
1856 }
1857 #[inline(always)]
1858 fn slide_u8x16<const SHIFT: usize>(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1859 if SHIFT >= 16usize {
1860 return b;
1861 }
1862 let result = dyn_alignr_128(
1863 self,
1864 Bytes::to_bytes(b).val.0,
1865 Bytes::to_bytes(a).val.0,
1866 SHIFT,
1867 );
1868 Bytes::from_bytes(u8x16 {
1869 val: crate::support::Aligned128(result),
1870 simd: self,
1871 })
1872 }
1873 #[inline(always)]
1874 fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1875 crate::kernel!(
1876 #[inline(always)]
1877 fn kernel(token: Avx512, a: u8x16<Avx512>, indices: u8x16<Avx512>) -> u8x16<Avx512> {
1878 let bytes = Bytes::to_bytes(a).val.0;
1879 let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into());
1880 Bytes::from_bytes(u8x16 {
1881 val: crate::support::Aligned128(result),
1882 simd: token,
1883 })
1884 }
1885 );
1886 kernel(self, a, indices)
1887 }
1888 #[inline(always)]
1889 fn swizzle_dyn_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1890 crate::kernel!(
1891 #[inline(always)]
1892 fn kernel(token: Avx512, a: u8x16<Avx512>, indices: u8x16<Avx512>) -> u8x16<Avx512> {
1893 let result = _mm_permutexvar_epi8(indices.into(), Bytes::to_bytes(a).val.0);
1894 Bytes::from_bytes(u8x16 {
1895 val: crate::support::Aligned128(result),
1896 simd: token,
1897 })
1898 }
1899 );
1900 kernel(self, a, indices)
1901 }
1902 #[inline(always)]
1903 fn swizzle_dyn_precise_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1904 crate::kernel!(
1905 #[inline(always)]
1906 fn kernel(token: Avx512, a: u8x16<Avx512>, indices: u8x16<Avx512>) -> u8x16<Avx512> {
1907 let bytes = Bytes::to_bytes(a).val.0;
1908 let indices = indices.into();
1909 let indices = _mm_min_epu8(indices, _mm_set1_epi8(16));
1910 let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128());
1911 let result_bytes = u8x16 {
1912 val: crate::support::Aligned128(result),
1913 simd: token,
1914 };
1915 Bytes::from_bytes(result_bytes)
1916 }
1917 );
1918 kernel(self, a, indices)
1919 }
1920 #[inline(always)]
1921 fn concat_swizzle_dyn_u8x16(
1922 self,
1923 a: u8x16<Self>,
1924 b: u8x16<Self>,
1925 indices: u8x16<Self>,
1926 ) -> u8x16<Self> {
1927 crate::kernel!(
1928 #[inline(always)]
1929 fn kernel(
1930 token: Avx512,
1931 a: u8x16<Avx512>,
1932 b: u8x16<Avx512>,
1933 indices: u8x16<Avx512>,
1934 ) -> u8x16<Avx512> {
1935 let result = _mm_permutex2var_epi8(
1936 Bytes::to_bytes(a).val.0,
1937 indices.into(),
1938 Bytes::to_bytes(b).val.0,
1939 );
1940 Bytes::from_bytes(u8x16 {
1941 val: crate::support::Aligned128(result),
1942 simd: token,
1943 })
1944 }
1945 );
1946 kernel(self, a, b, indices)
1947 }
1948 #[inline(always)]
1949 fn concat_swizzle_dyn_precise_u8x16(
1950 self,
1951 a: u8x16<Self>,
1952 b: u8x16<Self>,
1953 indices: u8x16<Self>,
1954 ) -> u8x16<Self> {
1955 crate::kernel!(
1956 #[inline(always)]
1957 fn kernel(
1958 token: Avx512,
1959 a: u8x16<Avx512>,
1960 b: u8x16<Avx512>,
1961 indices: u8x16<Avx512>,
1962 ) -> u8x16<Avx512> {
1963 let a_bytes = Bytes::to_bytes(a).val.0;
1964 let b_bytes = Bytes::to_bytes(b).val.0;
1965 let indices = indices.into();
1966 let in_range = _mm_cmplt_epu8_mask(indices, _mm_set1_epi8(32));
1967 let result = _mm_maskz_permutex2var_epi8(in_range, a_bytes, indices, b_bytes);
1968 let result_bytes = u8x16 {
1969 val: crate::support::Aligned128(result),
1970 simd: token,
1971 };
1972 Bytes::from_bytes(result_bytes)
1973 }
1974 );
1975 kernel(self, a, b, indices)
1976 }
1977 #[inline(always)]
1978 fn compress_u8x16(self, values: u8x16<Self>, mask: mask8x16<Self>) -> u8x16<Self> {
1979 crate::kernel!(
1980 #[inline(always)]
1981 fn kernel(
1982 token: Avx512,
1983 values: u8x16<Avx512>,
1984 mask: mask8x16<Avx512>,
1985 ) -> u8x16<Avx512> {
1986 _mm_maskz_compress_epi8(u64::from((mask).val) as u16, values.into())
1987 .simd_into(token)
1988 }
1989 );
1990 kernel(self, values, mask)
1991 }
1992 #[inline(always)]
1993 fn compress_merge_u8x16(
1994 self,
1995 values: u8x16<Self>,
1996 mask: mask8x16<Self>,
1997 merge: u8x16<Self>,
1998 ) -> u8x16<Self> {
1999 crate::kernel!(
2000 #[inline(always)]
2001 fn kernel(
2002 token: Avx512,
2003 values: u8x16<Avx512>,
2004 mask: mask8x16<Avx512>,
2005 merge: u8x16<Avx512>,
2006 ) -> u8x16<Avx512> {
2007 _mm_mask_compress_epi8(merge.into(), u64::from((mask).val) as u16, values.into())
2008 .simd_into(token)
2009 }
2010 );
2011 kernel(self, values, mask, merge)
2012 }
2013 #[inline(always)]
2014 fn expand_u8x16(self, values: u8x16<Self>, mask: mask8x16<Self>) -> u8x16<Self> {
2015 crate::kernel!(
2016 #[inline(always)]
2017 fn kernel(
2018 token: Avx512,
2019 values: u8x16<Avx512>,
2020 mask: mask8x16<Avx512>,
2021 ) -> u8x16<Avx512> {
2022 _mm_maskz_expand_epi8(u64::from((mask).val) as u16, values.into()).simd_into(token)
2023 }
2024 );
2025 kernel(self, values, mask)
2026 }
2027 #[inline(always)]
2028 fn expand_merge_u8x16(
2029 self,
2030 values: u8x16<Self>,
2031 mask: mask8x16<Self>,
2032 merge: u8x16<Self>,
2033 ) -> u8x16<Self> {
2034 crate::kernel!(
2035 #[inline(always)]
2036 fn kernel(
2037 token: Avx512,
2038 values: u8x16<Avx512>,
2039 mask: mask8x16<Avx512>,
2040 merge: u8x16<Avx512>,
2041 ) -> u8x16<Avx512> {
2042 _mm_mask_expand_epi8(merge.into(), u64::from((mask).val) as u16, values.into())
2043 .simd_into(token)
2044 }
2045 );
2046 kernel(self, values, mask, merge)
2047 }
2048 #[inline(always)]
2049 fn count_ones_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
2050 crate::kernel!(
2051 #[inline(always)]
2052 fn kernel(token: Avx512, a: u8x16<Avx512>) -> u8x16<Avx512> {
2053 _mm_popcnt_epi8(a.into()).simd_into(token)
2054 }
2055 );
2056 kernel(self, a)
2057 }
2058 #[inline(always)]
2059 fn count_zeros_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
2060 self.count_ones_u8x16(self.not_u8x16(a))
2061 }
2062 #[inline(always)]
2063 fn add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2064 crate::kernel!(
2065 #[inline(always)]
2066 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2067 _mm_add_epi8(a.into(), b.into()).simd_into(token)
2068 }
2069 );
2070 kernel(self, a, b)
2071 }
2072 #[inline(always)]
2073 fn saturating_add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2074 crate::kernel!(
2075 #[inline(always)]
2076 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2077 _mm_adds_epu8(a.into(), b.into()).simd_into(token)
2078 }
2079 );
2080 kernel(self, a, b)
2081 }
2082 #[inline(always)]
2083 fn sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2084 crate::kernel!(
2085 #[inline(always)]
2086 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2087 _mm_sub_epi8(a.into(), b.into()).simd_into(token)
2088 }
2089 );
2090 kernel(self, a, b)
2091 }
2092 #[inline(always)]
2093 fn saturating_sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2094 crate::kernel!(
2095 #[inline(always)]
2096 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2097 _mm_subs_epu8(a.into(), b.into()).simd_into(token)
2098 }
2099 );
2100 kernel(self, a, b)
2101 }
2102 #[inline(always)]
2103 fn mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2104 crate::kernel!(
2105 #[inline(always)]
2106 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2107 let a = a.into();
2108 let b = b.into();
2109 let low_mask = _mm_set1_epi16(0xFF);
2110 let dst_even = _mm_mullo_epi16(a, b);
2111 let dst_odd = _mm_maddubs_epi16(a, _mm_andnot_si128(low_mask, b));
2112 _mm_or_si128(
2113 _mm_slli_epi16(dst_odd, 8),
2114 _mm_and_si128(dst_even, low_mask),
2115 )
2116 .simd_into(token)
2117 }
2118 );
2119 kernel(self, a, b)
2120 }
2121 #[inline(always)]
2122 fn and_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2123 crate::kernel!(
2124 #[inline(always)]
2125 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2126 _mm_and_si128(a.into(), b.into()).simd_into(token)
2127 }
2128 );
2129 kernel(self, a, b)
2130 }
2131 #[inline(always)]
2132 fn or_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2133 crate::kernel!(
2134 #[inline(always)]
2135 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2136 _mm_or_si128(a.into(), b.into()).simd_into(token)
2137 }
2138 );
2139 kernel(self, a, b)
2140 }
2141 #[inline(always)]
2142 fn xor_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2143 crate::kernel!(
2144 #[inline(always)]
2145 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2146 _mm_xor_si128(a.into(), b.into()).simd_into(token)
2147 }
2148 );
2149 kernel(self, a, b)
2150 }
2151 #[inline(always)]
2152 fn not_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
2153 a ^ !0
2154 }
2155 #[inline(always)]
2156 fn shl_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
2157 crate::kernel!(
2158 #[inline(always)]
2159 fn kernel(token: Avx512, a: u8x16<Avx512>, shift: u32) -> u8x16<Avx512> {
2160 let val = a.into();
2161 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
2162 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
2163 let byte_mask = _mm_set1_epi8(mask_byte);
2164 _mm_sll_epi16(_mm_and_si128(val, byte_mask), shift_count).simd_into(token)
2165 }
2166 );
2167 kernel(self, a, shift)
2168 }
2169 #[inline(always)]
2170 fn shlv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2171 crate::kernel!(
2172 #[inline(always)]
2173 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2174 let val = a.into();
2175 let counts = b.into();
2176 let byte_mask = _mm_set1_epi16(0x00ff);
2177 let lo_counts = _mm_and_si128(counts, byte_mask);
2178 let hi_counts = _mm_srli_epi16::<8>(counts);
2179 let lo_shifted = _mm_sllv_epi16(val, lo_counts);
2180 let hi_values = _mm_andnot_si128(byte_mask, val);
2181 let hi_shifted = _mm_sllv_epi16(hi_values, hi_counts);
2182 _mm_mask_blend_epi8(0xaaaa_u16, lo_shifted, hi_shifted).simd_into(token)
2183 }
2184 );
2185 kernel(self, a, b)
2186 }
2187 #[inline(always)]
2188 fn shr_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
2189 crate::kernel!(
2190 #[inline(always)]
2191 fn kernel(token: Avx512, a: u8x16<Avx512>, shift: u32) -> u8x16<Avx512> {
2192 let val = a.into();
2193 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
2194 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
2195 let byte_mask = _mm_set1_epi8(mask_byte);
2196 let shifted = _mm_srl_epi16(val, shift_count);
2197 let result = { _mm_and_si128(shifted, byte_mask) };
2198 result.simd_into(token)
2199 }
2200 );
2201 kernel(self, a, shift)
2202 }
2203 #[inline(always)]
2204 fn shrv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2205 crate::kernel!(
2206 #[inline(always)]
2207 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2208 let val = a.into();
2209 let counts = b.into();
2210 let zero = _mm_setzero_si128();
2211 let value_extend = zero;
2212 let lo_values = _mm_unpacklo_epi8(val, value_extend);
2213 let hi_values = _mm_unpackhi_epi8(val, value_extend);
2214 let lo_counts = _mm_unpacklo_epi8(counts, zero);
2215 let hi_counts = _mm_unpackhi_epi8(counts, zero);
2216 let byte_mask = _mm_set1_epi16(0x00ff);
2217 let lo_shifted = _mm_and_si128(_mm_srlv_epi16(lo_values, lo_counts), byte_mask);
2218 let hi_shifted = _mm_and_si128(_mm_srlv_epi16(hi_values, hi_counts), byte_mask);
2219 _mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
2220 }
2221 );
2222 kernel(self, a, b)
2223 }
2224 #[inline(always)]
2225 fn reduce_max_u8x16(self, a: u8x16<Self>) -> u8 {
2226 crate::kernel!(
2227 #[inline(always)]
2228 fn kernel(token: Avx512, a: u8x16<Avx512>) -> u8 {
2229 let reduced: __m128i = a.into();
2230 let shifted = _mm_srli_si128::<8>(reduced);
2231 let reduced = { _mm_max_epu8(reduced, shifted) };
2232 let shifted = _mm_srli_si128::<4>(reduced);
2233 let reduced = { _mm_max_epu8(reduced, shifted) };
2234 let shifted = _mm_srli_si128::<2>(reduced);
2235 let reduced = { _mm_max_epu8(reduced, shifted) };
2236 let shifted = _mm_srli_si128::<1>(reduced);
2237 let reduced = { _mm_max_epu8(reduced, shifted) };
2238 {
2239 let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
2240 lanes[0]
2241 }
2242 }
2243 );
2244 kernel(self, a)
2245 }
2246 #[inline(always)]
2247 fn reduce_min_u8x16(self, a: u8x16<Self>) -> u8 {
2248 crate::kernel!(
2249 #[inline(always)]
2250 fn kernel(token: Avx512, a: u8x16<Avx512>) -> u8 {
2251 let reduced: __m128i = a.into();
2252 let shifted = _mm_srli_si128::<8>(reduced);
2253 let reduced = { _mm_min_epu8(reduced, shifted) };
2254 let shifted = _mm_srli_si128::<4>(reduced);
2255 let reduced = { _mm_min_epu8(reduced, shifted) };
2256 let shifted = _mm_srli_si128::<2>(reduced);
2257 let reduced = { _mm_min_epu8(reduced, shifted) };
2258 let shifted = _mm_srli_si128::<1>(reduced);
2259 let reduced = { _mm_min_epu8(reduced, shifted) };
2260 {
2261 let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
2262 lanes[0]
2263 }
2264 }
2265 );
2266 kernel(self, a)
2267 }
2268 #[inline(always)]
2269 fn reduce_sum_u8x16(self, a: u8x16<Self>) -> u8 {
2270 crate::kernel!(
2271 #[inline(always)]
2272 fn kernel(token: Avx512, a: u8x16<Avx512>) -> u8 {
2273 let sum: __m128i = a.into();
2274 let sum = _mm_add_epi8(sum, _mm_srli_si128::<8>(sum));
2275 let sum = _mm_add_epi8(sum, _mm_srli_si128::<4>(sum));
2276 let sum = _mm_add_epi8(sum, _mm_srli_si128::<2>(sum));
2277 let sum = _mm_add_epi8(sum, _mm_srli_si128::<1>(sum));
2278 let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&sum);
2279 lanes[0]
2280 }
2281 );
2282 kernel(self, a)
2283 }
2284 #[inline(always)]
2285 fn reduce_product_u8x16(self, a: u8x16<Self>) -> u8 {
2286 crate::kernel!(
2287 #[inline(always)]
2288 fn kernel(token: Avx512, a: u8x16<Avx512>) -> u8 {
2289 let value: __m128i = a.into();
2290 let product = _mm_mullo_epi16(
2291 _mm_unpacklo_epi8(value, value),
2292 _mm_unpackhi_epi8(value, value),
2293 );
2294 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
2295 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
2296 let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
2297 let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&product);
2298 lanes[0]
2299 }
2300 );
2301 kernel(self, a)
2302 }
2303 #[inline(always)]
2304 fn max_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2305 crate::kernel!(
2306 #[inline(always)]
2307 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2308 _mm_max_epu8(a.into(), b.into()).simd_into(token)
2309 }
2310 );
2311 kernel(self, a, b)
2312 }
2313 #[inline(always)]
2314 fn min_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2315 crate::kernel!(
2316 #[inline(always)]
2317 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2318 _mm_min_epu8(a.into(), b.into()).simd_into(token)
2319 }
2320 );
2321 kernel(self, a, b)
2322 }
2323 #[inline(always)]
2324 fn clamp_u8x16(self, a: u8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
2325 debug_assert!(
2326 b.simd_le(c).all_true(),
2327 "clamp requires min <= max for all elements"
2328 );
2329 a.max(b).min(c)
2330 }
2331 #[inline(always)]
2332 fn simd_eq_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2333 crate::kernel!(
2334 #[inline(always)]
2335 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> mask8x16<Avx512> {
2336 mask8x16 {
2337 val: _mm_cmpeq_epu8_mask(a.into(), b.into()),
2338 simd: token,
2339 }
2340 }
2341 );
2342 kernel(self, a, b)
2343 }
2344 #[inline(always)]
2345 fn simd_ne_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2346 crate::kernel!(
2347 #[inline(always)]
2348 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> mask8x16<Avx512> {
2349 mask8x16 {
2350 val: _mm_cmpneq_epu8_mask(a.into(), b.into()),
2351 simd: token,
2352 }
2353 }
2354 );
2355 kernel(self, a, b)
2356 }
2357 #[inline(always)]
2358 fn simd_lt_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2359 crate::kernel!(
2360 #[inline(always)]
2361 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> mask8x16<Avx512> {
2362 mask8x16 {
2363 val: _mm_cmplt_epu8_mask(a.into(), b.into()),
2364 simd: token,
2365 }
2366 }
2367 );
2368 kernel(self, a, b)
2369 }
2370 #[inline(always)]
2371 fn simd_le_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2372 crate::kernel!(
2373 #[inline(always)]
2374 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> mask8x16<Avx512> {
2375 mask8x16 {
2376 val: _mm_cmple_epu8_mask(a.into(), b.into()),
2377 simd: token,
2378 }
2379 }
2380 );
2381 kernel(self, a, b)
2382 }
2383 #[inline(always)]
2384 fn zip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2385 crate::kernel!(
2386 #[inline(always)]
2387 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2388 _mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
2389 }
2390 );
2391 kernel(self, a, b)
2392 }
2393 #[inline(always)]
2394 fn zip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2395 crate::kernel!(
2396 #[inline(always)]
2397 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2398 _mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
2399 }
2400 );
2401 kernel(self, a, b)
2402 }
2403 #[inline(always)]
2404 fn unzip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2405 crate::kernel!(
2406 #[inline(always)]
2407 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2408 _mm_permutex2var_epi8(
2409 a.into(),
2410 _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
2411 b.into(),
2412 )
2413 .simd_into(token)
2414 }
2415 );
2416 kernel(self, a, b)
2417 }
2418 #[inline(always)]
2419 fn unzip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2420 crate::kernel!(
2421 #[inline(always)]
2422 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
2423 _mm_permutex2var_epi8(
2424 a.into(),
2425 _mm_setr_epi8(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
2426 b.into(),
2427 )
2428 .simd_into(token)
2429 }
2430 );
2431 kernel(self, a, b)
2432 }
2433 #[inline(always)]
2434 fn interleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
2435 (self.zip_low_u8x16(a, b), self.zip_high_u8x16(a, b))
2436 }
2437 #[inline(always)]
2438 fn deinterleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
2439 crate::kernel!(
2440 #[inline(always)]
2441 fn kernel(
2442 token: Avx512,
2443 a: u8x16<Avx512>,
2444 b: u8x16<Avx512>,
2445 ) -> (u8x16<Avx512>, u8x16<Avx512>) {
2446 let a = a.into();
2447 let b = b.into();
2448 (
2449 _mm_permutex2var_epi8(
2450 a,
2451 _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
2452 b,
2453 )
2454 .simd_into(token),
2455 _mm_permutex2var_epi8(
2456 a,
2457 _mm_setr_epi8(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
2458 b,
2459 )
2460 .simd_into(token),
2461 )
2462 }
2463 );
2464 kernel(self, a, b)
2465 }
2466 #[inline(always)]
2467 fn select_u8x16(self, a: mask8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
2468 crate::kernel!(
2469 #[inline(always)]
2470 fn kernel(
2471 token: Avx512,
2472 a: mask8x16<Avx512>,
2473 b: u8x16<Avx512>,
2474 c: u8x16<Avx512>,
2475 ) -> u8x16<Avx512> {
2476 _mm_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token)
2477 }
2478 );
2479 kernel(self, a, b, c)
2480 }
2481 #[inline(always)]
2482 fn combine_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x32<Self> {
2483 crate::kernel!(
2484 #[inline(always)]
2485 fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x32<Avx512> {
2486 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
2487 }
2488 );
2489 kernel(self, a, b)
2490 }
2491 #[inline(always)]
2492 fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16<Self>; 4usize] {
2493 crate::kernel!(
2494 #[inline(always)]
2495 fn kernel(token: Avx512, src: &[u8; 64usize]) -> [u8x16<Avx512>; 4usize] {
2496 let lanes: __m512i =
2497 crate::transmute::checked_transmute_copy::<[u8; 64usize], __m512i>(src);
2498 let lanes = _mm512_permutexvar_epi8(
2499 _mm512_set_epi8(
2500 63, 59, 55, 51, 47, 43, 39, 35, 31, 27, 23, 19, 15, 11, 7, 3, 62, 58, 54,
2501 50, 46, 42, 38, 34, 30, 26, 22, 18, 14, 10, 6, 2, 61, 57, 53, 49, 45, 41,
2502 37, 33, 29, 25, 21, 17, 13, 9, 5, 1, 60, 56, 52, 48, 44, 40, 36, 32, 28,
2503 24, 20, 16, 12, 8, 4, 0,
2504 ),
2505 lanes,
2506 );
2507 let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes);
2508 [
2509 outputs[0].simd_into(token),
2510 outputs[1].simd_into(token),
2511 outputs[2].simd_into(token),
2512 outputs[3].simd_into(token),
2513 ]
2514 }
2515 );
2516 kernel(self, src)
2517 }
2518 #[inline(always)]
2519 fn store_four_interleaved_u8x16(
2520 self,
2521 vectors: [u8x16<Self>; 4usize],
2522 dest: &mut [u8; 64usize],
2523 ) -> () {
2524 crate::kernel!(
2525 #[inline(always)]
2526 fn kernel(
2527 token: Avx512,
2528 vectors: [u8x16<Avx512>; 4usize],
2529 dest: &mut [u8; 64usize],
2530 ) -> () {
2531 let _ = token;
2532 let inputs: [__m128i; 4] = [
2533 vectors[0].into(),
2534 vectors[1].into(),
2535 vectors[2].into(),
2536 vectors[3].into(),
2537 ];
2538 let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs);
2539 let lanes = _mm512_permutexvar_epi8(
2540 _mm512_set_epi8(
2541 63, 47, 31, 15, 62, 46, 30, 14, 61, 45, 29, 13, 60, 44, 28, 12, 59, 43, 27,
2542 11, 58, 42, 26, 10, 57, 41, 25, 9, 56, 40, 24, 8, 55, 39, 23, 7, 54, 38,
2543 22, 6, 53, 37, 21, 5, 52, 36, 20, 4, 51, 35, 19, 3, 50, 34, 18, 2, 49, 33,
2544 17, 1, 48, 32, 16, 0,
2545 ),
2546 lanes,
2547 );
2548 crate::transmute::checked_transmute_store::<__m512i, [u8; 64usize]>(lanes, dest);
2549 }
2550 );
2551 kernel(self, vectors, dest);
2552 }
2553 #[inline(always)]
2554 fn widen_u8x16(self, a: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2555 crate::kernel!(
2556 #[inline(always)]
2557 fn kernel(token: Avx512, a: u8x16<Avx512>) -> (u16x8<Avx512>, u16x8<Avx512>) {
2558 let raw = a.into();
2559 (
2560 _mm_cvtepu8_epi16(raw).simd_into(token),
2561 _mm_cvtepu8_epi16(_mm_srli_si128::<8>(raw)).simd_into(token),
2562 )
2563 }
2564 );
2565 kernel(self, a)
2566 }
2567 #[inline(always)]
2568 fn widening_mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2569 let (a0, a1) = self.widen_u8x16(a);
2570 let (b0, b1) = self.widen_u8x16(b);
2571 (self.mul_u16x8(a0, b0), self.mul_u16x8(a1, b1))
2572 }
2573 #[inline(always)]
2574 fn widening_add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2575 crate::kernel!(
2576 #[inline(always)]
2577 fn kernel(
2578 token: Avx512,
2579 a: u8x16<Avx512>,
2580 b: u8x16<Avx512>,
2581 ) -> (u16x8<Avx512>, u16x8<Avx512>) {
2582 let a = a.into();
2583 let b = b.into();
2584 let low = a;
2585 let high = b;
2586 let result_low = _mm_unpacklo_epi8(low, high);
2587 let result_high = _mm_unpackhi_epi8(low, high);
2588 let ones = _mm_set1_epi8(1);
2589 let result_low = _mm_maddubs_epi16(result_low, ones);
2590 let result_high = _mm_maddubs_epi16(result_high, ones);
2591 (result_low.simd_into(token), result_high.simd_into(token))
2592 }
2593 );
2594 kernel(self, a, b)
2595 }
2596 #[inline(always)]
2597 fn pairwise_widening_add_u8x16(self, a: u8x16<Self>) -> u16x8<Self> {
2598 crate::kernel!(
2599 #[inline(always)]
2600 fn kernel(token: Avx512, a: u8x16<Avx512>) -> u16x8<Avx512> {
2601 ({ _mm_maddubs_epi16(a.into(), _mm_set1_epi8(1)) }).simd_into(token)
2602 }
2603 );
2604 kernel(self, a)
2605 }
2606 #[inline(always)]
2607 fn pairwise_dot_product_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u16x8<Self> {
2608 let (low, high) = self.widening_mul_u8x16(a, b);
2609 let (even, odd) = self.deinterleave_u16x8(low, high);
2610 self.add_u16x8(even, odd)
2611 }
2612 #[inline(always)]
2613 fn splat_mask8x16(self, val: bool) -> mask8x16<Self> {
2614 mask8x16 {
2615 val: (if val { 65535u64 } else { 0 }) as _,
2616 simd: self,
2617 }
2618 }
2619 #[inline(always)]
2620 fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16<Self> {
2621 mask8x16 {
2622 val: (bits & 65535u64) as _,
2623 simd: self,
2624 }
2625 }
2626 #[inline(always)]
2627 fn to_bitmask_mask8x16(self, a: mask8x16<Self>) -> u64 {
2628 u64::from((a).val) & 65535u64
2629 }
2630 #[inline(always)]
2631 fn from_vector_mask8x16(self, vector: i8x16<Self>) -> mask8x16<Self> {
2632 crate::kernel!(
2633 #[inline(always)]
2634 fn kernel(token: Avx512, vector: i8x16<Avx512>) -> mask8x16<Avx512> {
2635 _mm_movepi8_mask(vector.into()).simd_into(token)
2636 }
2637 );
2638 kernel(self, vector)
2639 }
2640 #[inline(always)]
2641 fn to_vector_mask8x16(self, a: mask8x16<Self>) -> i8x16<Self> {
2642 crate::kernel!(
2643 #[inline(always)]
2644 fn kernel(token: Avx512, a: mask8x16<Avx512>) -> i8x16<Avx512> {
2645 _mm_movm_epi8(a.val).simd_into(token)
2646 }
2647 );
2648 kernel(self, a)
2649 }
2650 #[inline(always)]
2651 fn set_mask8x16(self, a: &mut mask8x16<Self>, index: usize, value: bool) -> () {
2652 assert!(
2653 index < 16usize,
2654 "mask lane index {index} is out of bounds for {} lanes",
2655 16usize
2656 );
2657 let bit = 1u64 << index;
2658 let bits = u64::from((a).val);
2659 let bits = if value { bits | bit } else { bits & !bit };
2660 *a = mask8x16 {
2661 val: (bits) as _,
2662 simd: self,
2663 };
2664 }
2665 #[inline(always)]
2666 fn rotate_elements_left_mask8x16<const OFFSET: usize>(
2667 self,
2668 a: mask8x16<Self>,
2669 ) -> mask8x16<Self> {
2670 mask8x16 {
2671 val: (a.val.rotate_right((OFFSET % 16) as u32)) as _,
2672 simd: self,
2673 }
2674 }
2675 #[inline(always)]
2676 fn rotate_elements_right_mask8x16<const OFFSET: usize>(
2677 self,
2678 a: mask8x16<Self>,
2679 ) -> mask8x16<Self> {
2680 mask8x16 {
2681 val: (a.val.rotate_left((OFFSET % 16) as u32)) as _,
2682 simd: self,
2683 }
2684 }
2685 #[inline(always)]
2686 fn and_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2687 mask8x16 {
2688 val: ((u64::from((a).val) & u64::from((b).val)) & 65535u64) as _,
2689 simd: self,
2690 }
2691 }
2692 #[inline(always)]
2693 fn or_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2694 mask8x16 {
2695 val: ((u64::from((a).val) | u64::from((b).val)) & 65535u64) as _,
2696 simd: self,
2697 }
2698 }
2699 #[inline(always)]
2700 fn xor_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2701 mask8x16 {
2702 val: ((u64::from((a).val) ^ u64::from((b).val)) & 65535u64) as _,
2703 simd: self,
2704 }
2705 }
2706 #[inline(always)]
2707 fn not_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
2708 mask8x16 {
2709 val: ((!u64::from((a).val)) & 65535u64) as _,
2710 simd: self,
2711 }
2712 }
2713 #[inline(always)]
2714 fn reverse_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
2715 mask8x16 {
2716 val: (a.val.reverse_bits() >> 0usize) as _,
2717 simd: self,
2718 }
2719 }
2720 #[inline(always)]
2721 fn select_mask8x16(
2722 self,
2723 a: mask8x16<Self>,
2724 b: mask8x16<Self>,
2725 c: mask8x16<Self>,
2726 ) -> mask8x16<Self> {
2727 mask8x16 {
2728 val: (((u64::from((a).val) & u64::from((b).val))
2729 | ((!u64::from((a).val)) & u64::from((c).val)))
2730 & 65535u64) as _,
2731 simd: self,
2732 }
2733 }
2734 #[inline(always)]
2735 fn simd_eq_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2736 mask8x16 {
2737 val: (!u64::from(a.val ^ b.val) & 65535u64) as _,
2738 simd: self,
2739 }
2740 }
2741 #[inline(always)]
2742 fn simd_ne_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2743 mask8x16 {
2744 val: (u64::from(a.val ^ b.val) & 65535u64) as _,
2745 simd: self,
2746 }
2747 }
2748 #[inline(always)]
2749 fn any_true_mask8x16(self, a: mask8x16<Self>) -> bool {
2750 let bits = u64::from((a).val) & 65535u64;
2751 bits != 0
2752 }
2753 #[inline(always)]
2754 fn all_true_mask8x16(self, a: mask8x16<Self>) -> bool {
2755 let bits = u64::from((a).val) & 65535u64;
2756 bits == 65535u64
2757 }
2758 #[inline(always)]
2759 fn any_false_mask8x16(self, a: mask8x16<Self>) -> bool {
2760 let bits = u64::from((a).val) & 65535u64;
2761 bits != 65535u64
2762 }
2763 #[inline(always)]
2764 fn all_false_mask8x16(self, a: mask8x16<Self>) -> bool {
2765 let bits = u64::from((a).val) & 65535u64;
2766 bits == 0
2767 }
2768 #[inline(always)]
2769 fn combine_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x32<Self> {
2770 let bits = (u64::from(a.val) | (u64::from(b.val) << 16usize)) & 4294967295u64;
2771 mask8x32 {
2772 val: bits as _,
2773 simd: self,
2774 }
2775 }
2776 #[inline(always)]
2777 fn widen_mask8x16(self, a: mask8x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
2778 let bits = u64::from(a.val);
2779 (
2780 mask16x8 {
2781 val: (bits & 255u64) as _,
2782 simd: self,
2783 },
2784 mask16x8 {
2785 val: ((bits >> 8usize) & 255u64) as _,
2786 simd: self,
2787 },
2788 )
2789 }
2790 #[inline(always)]
2791 fn abs_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2792 crate::kernel!(
2793 #[inline(always)]
2794 fn kernel(token: Avx512, a: i16x8<Avx512>) -> i16x8<Avx512> {
2795 _mm_abs_epi16(a.into()).simd_into(token)
2796 }
2797 );
2798 kernel(self, a)
2799 }
2800 #[inline(always)]
2801 fn splat_i16x8(self, val: i16) -> i16x8<Self> {
2802 crate::kernel!(
2803 #[inline(always)]
2804 fn kernel(token: Avx512, val: i16) -> i16x8<Avx512> {
2805 _mm_set1_epi16(val).simd_into(token)
2806 }
2807 );
2808 kernel(self, val)
2809 }
2810 #[inline(always)]
2811 fn reverse_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2812 let indices: u8x16<Self> =
2813 [14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1].simd_into(self);
2814 self.swizzle_dyn_i16x8(a, indices)
2815 }
2816 #[inline(always)]
2817 fn slide_i16x8<const SHIFT: usize>(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2818 if SHIFT >= 8usize {
2819 return b;
2820 }
2821 let result = dyn_alignr_128(
2822 self,
2823 Bytes::to_bytes(b).val.0,
2824 Bytes::to_bytes(a).val.0,
2825 SHIFT * 2usize,
2826 );
2827 Bytes::from_bytes(u8x16 {
2828 val: crate::support::Aligned128(result),
2829 simd: self,
2830 })
2831 }
2832 #[inline(always)]
2833 fn compress_i16x8(self, values: i16x8<Self>, mask: mask16x8<Self>) -> i16x8<Self> {
2834 crate::kernel!(
2835 #[inline(always)]
2836 fn kernel(
2837 token: Avx512,
2838 values: i16x8<Avx512>,
2839 mask: mask16x8<Avx512>,
2840 ) -> i16x8<Avx512> {
2841 _mm_maskz_compress_epi16(u64::from((mask).val) as u8, values.into())
2842 .simd_into(token)
2843 }
2844 );
2845 kernel(self, values, mask)
2846 }
2847 #[inline(always)]
2848 fn compress_merge_i16x8(
2849 self,
2850 values: i16x8<Self>,
2851 mask: mask16x8<Self>,
2852 merge: i16x8<Self>,
2853 ) -> i16x8<Self> {
2854 crate::kernel!(
2855 #[inline(always)]
2856 fn kernel(
2857 token: Avx512,
2858 values: i16x8<Avx512>,
2859 mask: mask16x8<Avx512>,
2860 merge: i16x8<Avx512>,
2861 ) -> i16x8<Avx512> {
2862 _mm_mask_compress_epi16(merge.into(), u64::from((mask).val) as u8, values.into())
2863 .simd_into(token)
2864 }
2865 );
2866 kernel(self, values, mask, merge)
2867 }
2868 #[inline(always)]
2869 fn expand_i16x8(self, values: i16x8<Self>, mask: mask16x8<Self>) -> i16x8<Self> {
2870 crate::kernel!(
2871 #[inline(always)]
2872 fn kernel(
2873 token: Avx512,
2874 values: i16x8<Avx512>,
2875 mask: mask16x8<Avx512>,
2876 ) -> i16x8<Avx512> {
2877 _mm_maskz_expand_epi16(u64::from((mask).val) as u8, values.into()).simd_into(token)
2878 }
2879 );
2880 kernel(self, values, mask)
2881 }
2882 #[inline(always)]
2883 fn expand_merge_i16x8(
2884 self,
2885 values: i16x8<Self>,
2886 mask: mask16x8<Self>,
2887 merge: i16x8<Self>,
2888 ) -> i16x8<Self> {
2889 crate::kernel!(
2890 #[inline(always)]
2891 fn kernel(
2892 token: Avx512,
2893 values: i16x8<Avx512>,
2894 mask: mask16x8<Avx512>,
2895 merge: i16x8<Avx512>,
2896 ) -> i16x8<Avx512> {
2897 _mm_mask_expand_epi16(merge.into(), u64::from((mask).val) as u8, values.into())
2898 .simd_into(token)
2899 }
2900 );
2901 kernel(self, values, mask, merge)
2902 }
2903 #[inline(always)]
2904 fn count_ones_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2905 crate::kernel!(
2906 #[inline(always)]
2907 fn kernel(token: Avx512, a: i16x8<Avx512>) -> i16x8<Avx512> {
2908 _mm_popcnt_epi16(a.into()).simd_into(token)
2909 }
2910 );
2911 kernel(self, a)
2912 }
2913 #[inline(always)]
2914 fn count_zeros_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2915 self.count_ones_i16x8(self.not_i16x8(a))
2916 }
2917 #[inline(always)]
2918 fn add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2919 crate::kernel!(
2920 #[inline(always)]
2921 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
2922 _mm_add_epi16(a.into(), b.into()).simd_into(token)
2923 }
2924 );
2925 kernel(self, a, b)
2926 }
2927 #[inline(always)]
2928 fn saturating_add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2929 crate::kernel!(
2930 #[inline(always)]
2931 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
2932 _mm_adds_epi16(a.into(), b.into()).simd_into(token)
2933 }
2934 );
2935 kernel(self, a, b)
2936 }
2937 #[inline(always)]
2938 fn sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2939 crate::kernel!(
2940 #[inline(always)]
2941 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
2942 _mm_sub_epi16(a.into(), b.into()).simd_into(token)
2943 }
2944 );
2945 kernel(self, a, b)
2946 }
2947 #[inline(always)]
2948 fn saturating_sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2949 crate::kernel!(
2950 #[inline(always)]
2951 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
2952 _mm_subs_epi16(a.into(), b.into()).simd_into(token)
2953 }
2954 );
2955 kernel(self, a, b)
2956 }
2957 #[inline(always)]
2958 fn mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2959 crate::kernel!(
2960 #[inline(always)]
2961 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
2962 _mm_mullo_epi16(a.into(), b.into()).simd_into(token)
2963 }
2964 );
2965 kernel(self, a, b)
2966 }
2967 #[inline(always)]
2968 fn and_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2969 crate::kernel!(
2970 #[inline(always)]
2971 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
2972 _mm_and_si128(a.into(), b.into()).simd_into(token)
2973 }
2974 );
2975 kernel(self, a, b)
2976 }
2977 #[inline(always)]
2978 fn or_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2979 crate::kernel!(
2980 #[inline(always)]
2981 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
2982 _mm_or_si128(a.into(), b.into()).simd_into(token)
2983 }
2984 );
2985 kernel(self, a, b)
2986 }
2987 #[inline(always)]
2988 fn xor_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2989 crate::kernel!(
2990 #[inline(always)]
2991 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
2992 _mm_xor_si128(a.into(), b.into()).simd_into(token)
2993 }
2994 );
2995 kernel(self, a, b)
2996 }
2997 #[inline(always)]
2998 fn not_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2999 a ^ !0
3000 }
3001 #[inline(always)]
3002 fn shl_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
3003 crate::kernel!(
3004 #[inline(always)]
3005 fn kernel(token: Avx512, a: i16x8<Avx512>, shift: u32) -> i16x8<Avx512> {
3006 _mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
3007 }
3008 );
3009 kernel(self, a, shift)
3010 }
3011 #[inline(always)]
3012 fn shlv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3013 crate::kernel!(
3014 #[inline(always)]
3015 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
3016 _mm_sllv_epi16(a.into(), b.into()).simd_into(token)
3017 }
3018 );
3019 kernel(self, a, b)
3020 }
3021 #[inline(always)]
3022 fn shr_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
3023 crate::kernel!(
3024 #[inline(always)]
3025 fn kernel(token: Avx512, a: i16x8<Avx512>, shift: u32) -> i16x8<Avx512> {
3026 _mm_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
3027 }
3028 );
3029 kernel(self, a, shift)
3030 }
3031 #[inline(always)]
3032 fn shrv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3033 crate::kernel!(
3034 #[inline(always)]
3035 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
3036 _mm_srav_epi16(a.into(), b.into()).simd_into(token)
3037 }
3038 );
3039 kernel(self, a, b)
3040 }
3041 #[inline(always)]
3042 fn reduce_max_i16x8(self, a: i16x8<Self>) -> i16 {
3043 crate::kernel!(
3044 #[inline(always)]
3045 fn kernel(token: Avx512, a: i16x8<Avx512>) -> i16 {
3046 let reduced: __m128i = a.into();
3047 let shifted = _mm_srli_si128::<8>(reduced);
3048 let reduced = { _mm_max_epi16(reduced, shifted) };
3049 let shifted = _mm_srli_si128::<4>(reduced);
3050 let reduced = { _mm_max_epi16(reduced, shifted) };
3051 let shifted = _mm_srli_si128::<2>(reduced);
3052 let reduced = { _mm_max_epi16(reduced, shifted) };
3053 {
3054 let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
3055 lanes[0]
3056 }
3057 }
3058 );
3059 kernel(self, a)
3060 }
3061 #[inline(always)]
3062 fn reduce_min_i16x8(self, a: i16x8<Self>) -> i16 {
3063 crate::kernel!(
3064 #[inline(always)]
3065 fn kernel(token: Avx512, a: i16x8<Avx512>) -> i16 {
3066 let reduced: __m128i = a.into();
3067 let shifted = _mm_srli_si128::<8>(reduced);
3068 let reduced = { _mm_min_epi16(reduced, shifted) };
3069 let shifted = _mm_srli_si128::<4>(reduced);
3070 let reduced = { _mm_min_epi16(reduced, shifted) };
3071 let shifted = _mm_srli_si128::<2>(reduced);
3072 let reduced = { _mm_min_epi16(reduced, shifted) };
3073 {
3074 let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
3075 lanes[0]
3076 }
3077 }
3078 );
3079 kernel(self, a)
3080 }
3081 #[inline(always)]
3082 fn reduce_sum_i16x8(self, a: i16x8<Self>) -> i16 {
3083 crate::kernel!(
3084 #[inline(always)]
3085 fn kernel(token: Avx512, a: i16x8<Avx512>) -> i16 {
3086 let sum: __m128i = a.into();
3087 let sum = _mm_add_epi16(sum, _mm_srli_si128::<8>(sum));
3088 let sum = _mm_add_epi16(sum, _mm_srli_si128::<4>(sum));
3089 let sum = _mm_add_epi16(sum, _mm_srli_si128::<2>(sum));
3090 let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&sum);
3091 lanes[0]
3092 }
3093 );
3094 kernel(self, a)
3095 }
3096 #[inline(always)]
3097 fn reduce_product_i16x8(self, a: i16x8<Self>) -> i16 {
3098 crate::kernel!(
3099 #[inline(always)]
3100 fn kernel(token: Avx512, a: i16x8<Avx512>) -> i16 {
3101 let product: __m128i = a.into();
3102 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
3103 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
3104 let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
3105 let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&product);
3106 lanes[0]
3107 }
3108 );
3109 kernel(self, a)
3110 }
3111 #[inline(always)]
3112 fn max_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3113 crate::kernel!(
3114 #[inline(always)]
3115 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
3116 _mm_max_epi16(a.into(), b.into()).simd_into(token)
3117 }
3118 );
3119 kernel(self, a, b)
3120 }
3121 #[inline(always)]
3122 fn min_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3123 crate::kernel!(
3124 #[inline(always)]
3125 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
3126 _mm_min_epi16(a.into(), b.into()).simd_into(token)
3127 }
3128 );
3129 kernel(self, a, b)
3130 }
3131 #[inline(always)]
3132 fn clamp_i16x8(self, a: i16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
3133 debug_assert!(
3134 b.simd_le(c).all_true(),
3135 "clamp requires min <= max for all elements"
3136 );
3137 a.max(b).min(c)
3138 }
3139 #[inline(always)]
3140 fn simd_eq_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
3141 crate::kernel!(
3142 #[inline(always)]
3143 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> mask16x8<Avx512> {
3144 mask16x8 {
3145 val: _mm_cmpeq_epi16_mask(a.into(), b.into()),
3146 simd: token,
3147 }
3148 }
3149 );
3150 kernel(self, a, b)
3151 }
3152 #[inline(always)]
3153 fn simd_ne_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
3154 crate::kernel!(
3155 #[inline(always)]
3156 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> mask16x8<Avx512> {
3157 mask16x8 {
3158 val: _mm_cmpneq_epi16_mask(a.into(), b.into()),
3159 simd: token,
3160 }
3161 }
3162 );
3163 kernel(self, a, b)
3164 }
3165 #[inline(always)]
3166 fn simd_lt_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
3167 crate::kernel!(
3168 #[inline(always)]
3169 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> mask16x8<Avx512> {
3170 mask16x8 {
3171 val: _mm_cmplt_epi16_mask(a.into(), b.into()),
3172 simd: token,
3173 }
3174 }
3175 );
3176 kernel(self, a, b)
3177 }
3178 #[inline(always)]
3179 fn simd_le_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
3180 crate::kernel!(
3181 #[inline(always)]
3182 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> mask16x8<Avx512> {
3183 mask16x8 {
3184 val: _mm_cmple_epi16_mask(a.into(), b.into()),
3185 simd: token,
3186 }
3187 }
3188 );
3189 kernel(self, a, b)
3190 }
3191 #[inline(always)]
3192 fn zip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3193 crate::kernel!(
3194 #[inline(always)]
3195 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
3196 _mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
3197 }
3198 );
3199 kernel(self, a, b)
3200 }
3201 #[inline(always)]
3202 fn zip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3203 crate::kernel!(
3204 #[inline(always)]
3205 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
3206 _mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
3207 }
3208 );
3209 kernel(self, a, b)
3210 }
3211 #[inline(always)]
3212 fn unzip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3213 crate::kernel!(
3214 #[inline(always)]
3215 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
3216 _mm_permutex2var_epi16(
3217 a.into(),
3218 _mm_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14),
3219 b.into(),
3220 )
3221 .simd_into(token)
3222 }
3223 );
3224 kernel(self, a, b)
3225 }
3226 #[inline(always)]
3227 fn unzip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3228 crate::kernel!(
3229 #[inline(always)]
3230 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
3231 _mm_permutex2var_epi16(
3232 a.into(),
3233 _mm_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15),
3234 b.into(),
3235 )
3236 .simd_into(token)
3237 }
3238 );
3239 kernel(self, a, b)
3240 }
3241 #[inline(always)]
3242 fn interleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
3243 (self.zip_low_i16x8(a, b), self.zip_high_i16x8(a, b))
3244 }
3245 #[inline(always)]
3246 fn deinterleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
3247 crate::kernel!(
3248 #[inline(always)]
3249 fn kernel(
3250 token: Avx512,
3251 a: i16x8<Avx512>,
3252 b: i16x8<Avx512>,
3253 ) -> (i16x8<Avx512>, i16x8<Avx512>) {
3254 let a = a.into();
3255 let b = b.into();
3256 (
3257 _mm_permutex2var_epi16(a, _mm_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14), b)
3258 .simd_into(token),
3259 _mm_permutex2var_epi16(a, _mm_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15), b)
3260 .simd_into(token),
3261 )
3262 }
3263 );
3264 kernel(self, a, b)
3265 }
3266 #[inline(always)]
3267 fn select_i16x8(self, a: mask16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
3268 crate::kernel!(
3269 #[inline(always)]
3270 fn kernel(
3271 token: Avx512,
3272 a: mask16x8<Avx512>,
3273 b: i16x8<Avx512>,
3274 c: i16x8<Avx512>,
3275 ) -> i16x8<Avx512> {
3276 _mm_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token)
3277 }
3278 );
3279 kernel(self, a, b, c)
3280 }
3281 #[inline(always)]
3282 fn combine_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x16<Self> {
3283 crate::kernel!(
3284 #[inline(always)]
3285 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x16<Avx512> {
3286 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
3287 }
3288 );
3289 kernel(self, a, b)
3290 }
3291 #[inline(always)]
3292 fn neg_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
3293 crate::kernel!(
3294 #[inline(always)]
3295 fn kernel(token: Avx512, a: i16x8<Avx512>) -> i16x8<Avx512> {
3296 _mm_sub_epi16(_mm_setzero_si128(), a.into()).simd_into(token)
3297 }
3298 );
3299 kernel(self, a)
3300 }
3301 #[inline(always)]
3302 fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8<Self>; 4usize] {
3303 crate::kernel!(
3304 #[inline(always)]
3305 fn kernel(token: Avx512, src: &[i16; 32usize]) -> [i16x8<Avx512>; 4usize] {
3306 let lanes: __m512i =
3307 crate::transmute::checked_transmute_copy::<[i16; 32usize], __m512i>(src);
3308 let lanes = _mm512_permutexvar_epi16(
3309 _mm512_set_epi16(
3310 31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2, 29, 25, 21, 17,
3311 13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0,
3312 ),
3313 lanes,
3314 );
3315 let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes);
3316 [
3317 outputs[0].simd_into(token),
3318 outputs[1].simd_into(token),
3319 outputs[2].simd_into(token),
3320 outputs[3].simd_into(token),
3321 ]
3322 }
3323 );
3324 kernel(self, src)
3325 }
3326 #[inline(always)]
3327 fn store_four_interleaved_i16x8(
3328 self,
3329 vectors: [i16x8<Self>; 4usize],
3330 dest: &mut [i16; 32usize],
3331 ) -> () {
3332 crate::kernel!(
3333 #[inline(always)]
3334 fn kernel(
3335 token: Avx512,
3336 vectors: [i16x8<Avx512>; 4usize],
3337 dest: &mut [i16; 32usize],
3338 ) -> () {
3339 let _ = token;
3340 let inputs: [__m128i; 4] = [
3341 vectors[0].into(),
3342 vectors[1].into(),
3343 vectors[2].into(),
3344 vectors[3].into(),
3345 ];
3346 let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs);
3347 let lanes = _mm512_permutexvar_epi16(
3348 _mm512_set_epi16(
3349 31, 23, 15, 7, 30, 22, 14, 6, 29, 21, 13, 5, 28, 20, 12, 4, 27, 19, 11, 3,
3350 26, 18, 10, 2, 25, 17, 9, 1, 24, 16, 8, 0,
3351 ),
3352 lanes,
3353 );
3354 crate::transmute::checked_transmute_store::<__m512i, [i16; 32usize]>(lanes, dest);
3355 }
3356 );
3357 kernel(self, vectors, dest);
3358 }
3359 #[inline(always)]
3360 fn widen_i16x8(self, a: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3361 crate::kernel!(
3362 #[inline(always)]
3363 fn kernel(token: Avx512, a: i16x8<Avx512>) -> (i32x4<Avx512>, i32x4<Avx512>) {
3364 let raw = a.into();
3365 (
3366 _mm_cvtepi16_epi32(raw).simd_into(token),
3367 _mm_cvtepi16_epi32(_mm_srli_si128::<8>(raw)).simd_into(token),
3368 )
3369 }
3370 );
3371 kernel(self, a)
3372 }
3373 #[inline(always)]
3374 fn widening_mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3375 crate::kernel!(
3376 #[inline(always)]
3377 fn kernel(
3378 token: Avx512,
3379 a: i16x8<Avx512>,
3380 b: i16x8<Avx512>,
3381 ) -> (i32x4<Avx512>, i32x4<Avx512>) {
3382 let a = a.into();
3383 let b = b.into();
3384 let low = _mm_mullo_epi16(a, b);
3385 let high = _mm_mulhi_epi16(a, b);
3386 let result_low = _mm_unpacklo_epi16(low, high);
3387 let result_high = _mm_unpackhi_epi16(low, high);
3388 (result_low.simd_into(token), result_high.simd_into(token))
3389 }
3390 );
3391 kernel(self, a, b)
3392 }
3393 #[inline(always)]
3394 fn widening_add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3395 crate::kernel!(
3396 #[inline(always)]
3397 fn kernel(
3398 token: Avx512,
3399 a: i16x8<Avx512>,
3400 b: i16x8<Avx512>,
3401 ) -> (i32x4<Avx512>, i32x4<Avx512>) {
3402 let a = a.into();
3403 let b = b.into();
3404 let low = a;
3405 let high = b;
3406 let result_low = _mm_unpacklo_epi16(low, high);
3407 let result_high = _mm_unpackhi_epi16(low, high);
3408 let ones = _mm_set1_epi16(1);
3409 let result_low = _mm_madd_epi16(result_low, ones);
3410 let result_high = _mm_madd_epi16(result_high, ones);
3411 (result_low.simd_into(token), result_high.simd_into(token))
3412 }
3413 );
3414 kernel(self, a, b)
3415 }
3416 #[inline(always)]
3417 fn pairwise_widening_add_i16x8(self, a: i16x8<Self>) -> i32x4<Self> {
3418 crate::kernel!(
3419 #[inline(always)]
3420 fn kernel(token: Avx512, a: i16x8<Avx512>) -> i32x4<Avx512> {
3421 ({ _mm_madd_epi16(a.into(), _mm_set1_epi16(1)) }).simd_into(token)
3422 }
3423 );
3424 kernel(self, a)
3425 }
3426 #[inline(always)]
3427 fn pairwise_dot_product_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i32x4<Self> {
3428 crate::kernel!(
3429 #[inline(always)]
3430 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i32x4<Avx512> {
3431 _mm_madd_epi16(a.into(), b.into()).simd_into(token)
3432 }
3433 );
3434 kernel(self, a, b)
3435 }
3436 #[inline(always)]
3437 fn narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3438 crate::kernel!(
3439 #[inline(always)]
3440 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i8x16<Avx512> {
3441 let low = _mm_cvtepi16_epi8(a.into());
3442 let high = _mm_cvtepi16_epi8(b.into());
3443 _mm_unpacklo_epi64(low, high).simd_into(token)
3444 }
3445 );
3446 kernel(self, a, b)
3447 }
3448 #[inline(always)]
3449 fn saturating_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3450 crate::kernel!(
3451 #[inline(always)]
3452 fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i8x16<Avx512> {
3453 _mm_packs_epi16(a.into(), b.into()).simd_into(token)
3454 }
3455 );
3456 kernel(self, a, b)
3457 }
3458 #[inline(always)]
3459 fn relaxed_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3460 debug_assert!(
3461 a.as_slice()
3462 .iter()
3463 .chain(b.as_slice())
3464 .all(|&value| { value >= i8::MIN as i16 && value <= i8::MAX as i16 }),
3465 "relaxed_narrow inputs must fit in the destination type",
3466 );
3467 self.saturating_narrow_i16x8(a, b)
3468 }
3469 #[inline(always)]
3470 fn splat_u16x8(self, val: u16) -> u16x8<Self> {
3471 crate::kernel!(
3472 #[inline(always)]
3473 fn kernel(token: Avx512, val: u16) -> u16x8<Avx512> {
3474 _mm_set1_epi16(val.cast_signed()).simd_into(token)
3475 }
3476 );
3477 kernel(self, val)
3478 }
3479 #[inline(always)]
3480 fn reverse_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3481 let indices: u8x16<Self> =
3482 [14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1].simd_into(self);
3483 self.swizzle_dyn_u16x8(a, indices)
3484 }
3485 #[inline(always)]
3486 fn slide_u16x8<const SHIFT: usize>(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3487 if SHIFT >= 8usize {
3488 return b;
3489 }
3490 let result = dyn_alignr_128(
3491 self,
3492 Bytes::to_bytes(b).val.0,
3493 Bytes::to_bytes(a).val.0,
3494 SHIFT * 2usize,
3495 );
3496 Bytes::from_bytes(u8x16 {
3497 val: crate::support::Aligned128(result),
3498 simd: self,
3499 })
3500 }
3501 #[inline(always)]
3502 fn compress_u16x8(self, values: u16x8<Self>, mask: mask16x8<Self>) -> u16x8<Self> {
3503 crate::kernel!(
3504 #[inline(always)]
3505 fn kernel(
3506 token: Avx512,
3507 values: u16x8<Avx512>,
3508 mask: mask16x8<Avx512>,
3509 ) -> u16x8<Avx512> {
3510 _mm_maskz_compress_epi16(u64::from((mask).val) as u8, values.into())
3511 .simd_into(token)
3512 }
3513 );
3514 kernel(self, values, mask)
3515 }
3516 #[inline(always)]
3517 fn compress_merge_u16x8(
3518 self,
3519 values: u16x8<Self>,
3520 mask: mask16x8<Self>,
3521 merge: u16x8<Self>,
3522 ) -> u16x8<Self> {
3523 crate::kernel!(
3524 #[inline(always)]
3525 fn kernel(
3526 token: Avx512,
3527 values: u16x8<Avx512>,
3528 mask: mask16x8<Avx512>,
3529 merge: u16x8<Avx512>,
3530 ) -> u16x8<Avx512> {
3531 _mm_mask_compress_epi16(merge.into(), u64::from((mask).val) as u8, values.into())
3532 .simd_into(token)
3533 }
3534 );
3535 kernel(self, values, mask, merge)
3536 }
3537 #[inline(always)]
3538 fn expand_u16x8(self, values: u16x8<Self>, mask: mask16x8<Self>) -> u16x8<Self> {
3539 crate::kernel!(
3540 #[inline(always)]
3541 fn kernel(
3542 token: Avx512,
3543 values: u16x8<Avx512>,
3544 mask: mask16x8<Avx512>,
3545 ) -> u16x8<Avx512> {
3546 _mm_maskz_expand_epi16(u64::from((mask).val) as u8, values.into()).simd_into(token)
3547 }
3548 );
3549 kernel(self, values, mask)
3550 }
3551 #[inline(always)]
3552 fn expand_merge_u16x8(
3553 self,
3554 values: u16x8<Self>,
3555 mask: mask16x8<Self>,
3556 merge: u16x8<Self>,
3557 ) -> u16x8<Self> {
3558 crate::kernel!(
3559 #[inline(always)]
3560 fn kernel(
3561 token: Avx512,
3562 values: u16x8<Avx512>,
3563 mask: mask16x8<Avx512>,
3564 merge: u16x8<Avx512>,
3565 ) -> u16x8<Avx512> {
3566 _mm_mask_expand_epi16(merge.into(), u64::from((mask).val) as u8, values.into())
3567 .simd_into(token)
3568 }
3569 );
3570 kernel(self, values, mask, merge)
3571 }
3572 #[inline(always)]
3573 fn count_ones_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3574 crate::kernel!(
3575 #[inline(always)]
3576 fn kernel(token: Avx512, a: u16x8<Avx512>) -> u16x8<Avx512> {
3577 _mm_popcnt_epi16(a.into()).simd_into(token)
3578 }
3579 );
3580 kernel(self, a)
3581 }
3582 #[inline(always)]
3583 fn count_zeros_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3584 self.count_ones_u16x8(self.not_u16x8(a))
3585 }
3586 #[inline(always)]
3587 fn add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3588 crate::kernel!(
3589 #[inline(always)]
3590 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3591 _mm_add_epi16(a.into(), b.into()).simd_into(token)
3592 }
3593 );
3594 kernel(self, a, b)
3595 }
3596 #[inline(always)]
3597 fn saturating_add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3598 crate::kernel!(
3599 #[inline(always)]
3600 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3601 _mm_adds_epu16(a.into(), b.into()).simd_into(token)
3602 }
3603 );
3604 kernel(self, a, b)
3605 }
3606 #[inline(always)]
3607 fn sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3608 crate::kernel!(
3609 #[inline(always)]
3610 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3611 _mm_sub_epi16(a.into(), b.into()).simd_into(token)
3612 }
3613 );
3614 kernel(self, a, b)
3615 }
3616 #[inline(always)]
3617 fn saturating_sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3618 crate::kernel!(
3619 #[inline(always)]
3620 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3621 _mm_subs_epu16(a.into(), b.into()).simd_into(token)
3622 }
3623 );
3624 kernel(self, a, b)
3625 }
3626 #[inline(always)]
3627 fn mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3628 crate::kernel!(
3629 #[inline(always)]
3630 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3631 _mm_mullo_epi16(a.into(), b.into()).simd_into(token)
3632 }
3633 );
3634 kernel(self, a, b)
3635 }
3636 #[inline(always)]
3637 fn and_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3638 crate::kernel!(
3639 #[inline(always)]
3640 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3641 _mm_and_si128(a.into(), b.into()).simd_into(token)
3642 }
3643 );
3644 kernel(self, a, b)
3645 }
3646 #[inline(always)]
3647 fn or_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3648 crate::kernel!(
3649 #[inline(always)]
3650 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3651 _mm_or_si128(a.into(), b.into()).simd_into(token)
3652 }
3653 );
3654 kernel(self, a, b)
3655 }
3656 #[inline(always)]
3657 fn xor_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3658 crate::kernel!(
3659 #[inline(always)]
3660 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3661 _mm_xor_si128(a.into(), b.into()).simd_into(token)
3662 }
3663 );
3664 kernel(self, a, b)
3665 }
3666 #[inline(always)]
3667 fn not_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3668 a ^ !0
3669 }
3670 #[inline(always)]
3671 fn shl_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
3672 crate::kernel!(
3673 #[inline(always)]
3674 fn kernel(token: Avx512, a: u16x8<Avx512>, shift: u32) -> u16x8<Avx512> {
3675 _mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
3676 }
3677 );
3678 kernel(self, a, shift)
3679 }
3680 #[inline(always)]
3681 fn shlv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3682 crate::kernel!(
3683 #[inline(always)]
3684 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3685 _mm_sllv_epi16(a.into(), b.into()).simd_into(token)
3686 }
3687 );
3688 kernel(self, a, b)
3689 }
3690 #[inline(always)]
3691 fn shr_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
3692 crate::kernel!(
3693 #[inline(always)]
3694 fn kernel(token: Avx512, a: u16x8<Avx512>, shift: u32) -> u16x8<Avx512> {
3695 _mm_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
3696 }
3697 );
3698 kernel(self, a, shift)
3699 }
3700 #[inline(always)]
3701 fn shrv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3702 crate::kernel!(
3703 #[inline(always)]
3704 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3705 _mm_srlv_epi16(a.into(), b.into()).simd_into(token)
3706 }
3707 );
3708 kernel(self, a, b)
3709 }
3710 #[inline(always)]
3711 fn reduce_max_u16x8(self, a: u16x8<Self>) -> u16 {
3712 crate::kernel!(
3713 #[inline(always)]
3714 fn kernel(token: Avx512, a: u16x8<Avx512>) -> u16 {
3715 let reduced: __m128i = a.into();
3716 let shifted = _mm_srli_si128::<8>(reduced);
3717 let reduced = { _mm_max_epu16(reduced, shifted) };
3718 let shifted = _mm_srli_si128::<4>(reduced);
3719 let reduced = { _mm_max_epu16(reduced, shifted) };
3720 let shifted = _mm_srli_si128::<2>(reduced);
3721 let reduced = { _mm_max_epu16(reduced, shifted) };
3722 {
3723 let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
3724 lanes[0]
3725 }
3726 }
3727 );
3728 kernel(self, a)
3729 }
3730 #[inline(always)]
3731 fn reduce_min_u16x8(self, a: u16x8<Self>) -> u16 {
3732 crate::kernel!(
3733 #[inline(always)]
3734 fn kernel(token: Avx512, a: u16x8<Avx512>) -> u16 {
3735 let reduced: __m128i = a.into();
3736 let shifted = _mm_srli_si128::<8>(reduced);
3737 let reduced = { _mm_min_epu16(reduced, shifted) };
3738 let shifted = _mm_srli_si128::<4>(reduced);
3739 let reduced = { _mm_min_epu16(reduced, shifted) };
3740 let shifted = _mm_srli_si128::<2>(reduced);
3741 let reduced = { _mm_min_epu16(reduced, shifted) };
3742 {
3743 let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
3744 lanes[0]
3745 }
3746 }
3747 );
3748 kernel(self, a)
3749 }
3750 #[inline(always)]
3751 fn reduce_sum_u16x8(self, a: u16x8<Self>) -> u16 {
3752 crate::kernel!(
3753 #[inline(always)]
3754 fn kernel(token: Avx512, a: u16x8<Avx512>) -> u16 {
3755 let sum: __m128i = a.into();
3756 let sum = _mm_add_epi16(sum, _mm_srli_si128::<8>(sum));
3757 let sum = _mm_add_epi16(sum, _mm_srli_si128::<4>(sum));
3758 let sum = _mm_add_epi16(sum, _mm_srli_si128::<2>(sum));
3759 let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&sum);
3760 lanes[0]
3761 }
3762 );
3763 kernel(self, a)
3764 }
3765 #[inline(always)]
3766 fn reduce_product_u16x8(self, a: u16x8<Self>) -> u16 {
3767 crate::kernel!(
3768 #[inline(always)]
3769 fn kernel(token: Avx512, a: u16x8<Avx512>) -> u16 {
3770 let product: __m128i = a.into();
3771 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
3772 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
3773 let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
3774 let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&product);
3775 lanes[0]
3776 }
3777 );
3778 kernel(self, a)
3779 }
3780 #[inline(always)]
3781 fn max_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3782 crate::kernel!(
3783 #[inline(always)]
3784 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3785 _mm_max_epu16(a.into(), b.into()).simd_into(token)
3786 }
3787 );
3788 kernel(self, a, b)
3789 }
3790 #[inline(always)]
3791 fn min_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3792 crate::kernel!(
3793 #[inline(always)]
3794 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3795 _mm_min_epu16(a.into(), b.into()).simd_into(token)
3796 }
3797 );
3798 kernel(self, a, b)
3799 }
3800 #[inline(always)]
3801 fn clamp_u16x8(self, a: u16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
3802 debug_assert!(
3803 b.simd_le(c).all_true(),
3804 "clamp requires min <= max for all elements"
3805 );
3806 a.max(b).min(c)
3807 }
3808 #[inline(always)]
3809 fn simd_eq_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
3810 crate::kernel!(
3811 #[inline(always)]
3812 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> mask16x8<Avx512> {
3813 mask16x8 {
3814 val: _mm_cmpeq_epu16_mask(a.into(), b.into()),
3815 simd: token,
3816 }
3817 }
3818 );
3819 kernel(self, a, b)
3820 }
3821 #[inline(always)]
3822 fn simd_ne_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
3823 crate::kernel!(
3824 #[inline(always)]
3825 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> mask16x8<Avx512> {
3826 mask16x8 {
3827 val: _mm_cmpneq_epu16_mask(a.into(), b.into()),
3828 simd: token,
3829 }
3830 }
3831 );
3832 kernel(self, a, b)
3833 }
3834 #[inline(always)]
3835 fn simd_lt_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
3836 crate::kernel!(
3837 #[inline(always)]
3838 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> mask16x8<Avx512> {
3839 mask16x8 {
3840 val: _mm_cmplt_epu16_mask(a.into(), b.into()),
3841 simd: token,
3842 }
3843 }
3844 );
3845 kernel(self, a, b)
3846 }
3847 #[inline(always)]
3848 fn simd_le_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
3849 crate::kernel!(
3850 #[inline(always)]
3851 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> mask16x8<Avx512> {
3852 mask16x8 {
3853 val: _mm_cmple_epu16_mask(a.into(), b.into()),
3854 simd: token,
3855 }
3856 }
3857 );
3858 kernel(self, a, b)
3859 }
3860 #[inline(always)]
3861 fn zip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3862 crate::kernel!(
3863 #[inline(always)]
3864 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3865 _mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
3866 }
3867 );
3868 kernel(self, a, b)
3869 }
3870 #[inline(always)]
3871 fn zip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3872 crate::kernel!(
3873 #[inline(always)]
3874 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3875 _mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
3876 }
3877 );
3878 kernel(self, a, b)
3879 }
3880 #[inline(always)]
3881 fn unzip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3882 crate::kernel!(
3883 #[inline(always)]
3884 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3885 _mm_permutex2var_epi16(
3886 a.into(),
3887 _mm_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14),
3888 b.into(),
3889 )
3890 .simd_into(token)
3891 }
3892 );
3893 kernel(self, a, b)
3894 }
3895 #[inline(always)]
3896 fn unzip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3897 crate::kernel!(
3898 #[inline(always)]
3899 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
3900 _mm_permutex2var_epi16(
3901 a.into(),
3902 _mm_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15),
3903 b.into(),
3904 )
3905 .simd_into(token)
3906 }
3907 );
3908 kernel(self, a, b)
3909 }
3910 #[inline(always)]
3911 fn interleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
3912 (self.zip_low_u16x8(a, b), self.zip_high_u16x8(a, b))
3913 }
3914 #[inline(always)]
3915 fn deinterleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
3916 crate::kernel!(
3917 #[inline(always)]
3918 fn kernel(
3919 token: Avx512,
3920 a: u16x8<Avx512>,
3921 b: u16x8<Avx512>,
3922 ) -> (u16x8<Avx512>, u16x8<Avx512>) {
3923 let a = a.into();
3924 let b = b.into();
3925 (
3926 _mm_permutex2var_epi16(a, _mm_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14), b)
3927 .simd_into(token),
3928 _mm_permutex2var_epi16(a, _mm_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15), b)
3929 .simd_into(token),
3930 )
3931 }
3932 );
3933 kernel(self, a, b)
3934 }
3935 #[inline(always)]
3936 fn select_u16x8(self, a: mask16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
3937 crate::kernel!(
3938 #[inline(always)]
3939 fn kernel(
3940 token: Avx512,
3941 a: mask16x8<Avx512>,
3942 b: u16x8<Avx512>,
3943 c: u16x8<Avx512>,
3944 ) -> u16x8<Avx512> {
3945 _mm_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token)
3946 }
3947 );
3948 kernel(self, a, b, c)
3949 }
3950 #[inline(always)]
3951 fn combine_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x16<Self> {
3952 crate::kernel!(
3953 #[inline(always)]
3954 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x16<Avx512> {
3955 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
3956 }
3957 );
3958 kernel(self, a, b)
3959 }
3960 #[inline(always)]
3961 fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8<Self>; 4usize] {
3962 crate::kernel!(
3963 #[inline(always)]
3964 fn kernel(token: Avx512, src: &[u16; 32usize]) -> [u16x8<Avx512>; 4usize] {
3965 let lanes: __m512i =
3966 crate::transmute::checked_transmute_copy::<[u16; 32usize], __m512i>(src);
3967 let lanes = _mm512_permutexvar_epi16(
3968 _mm512_set_epi16(
3969 31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2, 29, 25, 21, 17,
3970 13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0,
3971 ),
3972 lanes,
3973 );
3974 let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes);
3975 [
3976 outputs[0].simd_into(token),
3977 outputs[1].simd_into(token),
3978 outputs[2].simd_into(token),
3979 outputs[3].simd_into(token),
3980 ]
3981 }
3982 );
3983 kernel(self, src)
3984 }
3985 #[inline(always)]
3986 fn store_four_interleaved_u16x8(
3987 self,
3988 vectors: [u16x8<Self>; 4usize],
3989 dest: &mut [u16; 32usize],
3990 ) -> () {
3991 crate::kernel!(
3992 #[inline(always)]
3993 fn kernel(
3994 token: Avx512,
3995 vectors: [u16x8<Avx512>; 4usize],
3996 dest: &mut [u16; 32usize],
3997 ) -> () {
3998 let _ = token;
3999 let inputs: [__m128i; 4] = [
4000 vectors[0].into(),
4001 vectors[1].into(),
4002 vectors[2].into(),
4003 vectors[3].into(),
4004 ];
4005 let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs);
4006 let lanes = _mm512_permutexvar_epi16(
4007 _mm512_set_epi16(
4008 31, 23, 15, 7, 30, 22, 14, 6, 29, 21, 13, 5, 28, 20, 12, 4, 27, 19, 11, 3,
4009 26, 18, 10, 2, 25, 17, 9, 1, 24, 16, 8, 0,
4010 ),
4011 lanes,
4012 );
4013 crate::transmute::checked_transmute_store::<__m512i, [u16; 32usize]>(lanes, dest);
4014 }
4015 );
4016 kernel(self, vectors, dest);
4017 }
4018 #[inline(always)]
4019 fn widen_u16x8(self, a: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
4020 crate::kernel!(
4021 #[inline(always)]
4022 fn kernel(token: Avx512, a: u16x8<Avx512>) -> (u32x4<Avx512>, u32x4<Avx512>) {
4023 let raw = a.into();
4024 (
4025 _mm_cvtepu16_epi32(raw).simd_into(token),
4026 _mm_cvtepu16_epi32(_mm_srli_si128::<8>(raw)).simd_into(token),
4027 )
4028 }
4029 );
4030 kernel(self, a)
4031 }
4032 #[inline(always)]
4033 fn widening_mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
4034 crate::kernel!(
4035 #[inline(always)]
4036 fn kernel(
4037 token: Avx512,
4038 a: u16x8<Avx512>,
4039 b: u16x8<Avx512>,
4040 ) -> (u32x4<Avx512>, u32x4<Avx512>) {
4041 let a = a.into();
4042 let b = b.into();
4043 let low = _mm_mullo_epi16(a, b);
4044 let high = _mm_mulhi_epu16(a, b);
4045 let result_low = _mm_unpacklo_epi16(low, high);
4046 let result_high = _mm_unpackhi_epi16(low, high);
4047 (result_low.simd_into(token), result_high.simd_into(token))
4048 }
4049 );
4050 kernel(self, a, b)
4051 }
4052 #[inline(always)]
4053 fn widening_add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
4054 let (a0, a1) = self.widen_u16x8(a);
4055 let (b0, b1) = self.widen_u16x8(b);
4056 (self.add_u32x4(a0, b0), self.add_u32x4(a1, b1))
4057 }
4058 #[inline(always)]
4059 fn pairwise_widening_add_u16x8(self, a: u16x8<Self>) -> u32x4<Self> {
4060 crate::kernel!(
4061 #[inline(always)]
4062 fn kernel(token: Avx512, a: u16x8<Avx512>) -> u32x4<Avx512> {
4063 ({
4064 let a = a.into();
4065 _mm_add_epi32(
4066 _mm_and_si128(a, _mm_set1_epi32(65535)),
4067 _mm_srli_epi32::<16>(a),
4068 )
4069 })
4070 .simd_into(token)
4071 }
4072 );
4073 kernel(self, a)
4074 }
4075 #[inline(always)]
4076 fn pairwise_dot_product_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u32x4<Self> {
4077 let (low, high) = self.widening_mul_u16x8(a, b);
4078 let (even, odd) = self.deinterleave_u32x4(low, high);
4079 self.add_u32x4(even, odd)
4080 }
4081 #[inline(always)]
4082 fn narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
4083 crate::kernel!(
4084 #[inline(always)]
4085 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u8x16<Avx512> {
4086 let low = _mm_cvtepi16_epi8(a.into());
4087 let high = _mm_cvtepi16_epi8(b.into());
4088 _mm_unpacklo_epi64(low, high).simd_into(token)
4089 }
4090 );
4091 kernel(self, a, b)
4092 }
4093 #[inline(always)]
4094 fn saturating_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
4095 crate::kernel!(
4096 #[inline(always)]
4097 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u8x16<Avx512> {
4098 {
4099 let max = _mm_set1_epi16(i16::MAX);
4100 _mm_packus_epi16(_mm_min_epu16(a.into(), max), _mm_min_epu16(b.into(), max))
4101 }
4102 .simd_into(token)
4103 }
4104 );
4105 kernel(self, a, b)
4106 }
4107 #[inline(always)]
4108 fn relaxed_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
4109 crate::kernel!(
4110 #[inline(always)]
4111 fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u8x16<Avx512> {
4112 debug_assert!(
4113 a.as_slice()
4114 .iter()
4115 .chain(b.as_slice())
4116 .all(|&value| { value >= u8::MIN as u16 && value <= u8::MAX as u16 }),
4117 "relaxed_narrow inputs must fit in the destination type",
4118 );
4119 _mm_packus_epi16(a.into(), b.into()).simd_into(token)
4120 }
4121 );
4122 kernel(self, a, b)
4123 }
4124 #[inline(always)]
4125 fn splat_mask16x8(self, val: bool) -> mask16x8<Self> {
4126 mask16x8 {
4127 val: (if val { 255u64 } else { 0 }) as _,
4128 simd: self,
4129 }
4130 }
4131 #[inline(always)]
4132 fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8<Self> {
4133 mask16x8 {
4134 val: (bits & 255u64) as _,
4135 simd: self,
4136 }
4137 }
4138 #[inline(always)]
4139 fn to_bitmask_mask16x8(self, a: mask16x8<Self>) -> u64 {
4140 u64::from((a).val) & 255u64
4141 }
4142 #[inline(always)]
4143 fn from_vector_mask16x8(self, vector: i16x8<Self>) -> mask16x8<Self> {
4144 crate::kernel!(
4145 #[inline(always)]
4146 fn kernel(token: Avx512, vector: i16x8<Avx512>) -> mask16x8<Avx512> {
4147 _mm_movepi16_mask(vector.into()).simd_into(token)
4148 }
4149 );
4150 kernel(self, vector)
4151 }
4152 #[inline(always)]
4153 fn to_vector_mask16x8(self, a: mask16x8<Self>) -> i16x8<Self> {
4154 crate::kernel!(
4155 #[inline(always)]
4156 fn kernel(token: Avx512, a: mask16x8<Avx512>) -> i16x8<Avx512> {
4157 _mm_movm_epi16(a.val).simd_into(token)
4158 }
4159 );
4160 kernel(self, a)
4161 }
4162 #[inline(always)]
4163 fn set_mask16x8(self, a: &mut mask16x8<Self>, index: usize, value: bool) -> () {
4164 assert!(
4165 index < 8usize,
4166 "mask lane index {index} is out of bounds for {} lanes",
4167 8usize
4168 );
4169 let bit = 1u64 << index;
4170 let bits = u64::from((a).val);
4171 let bits = if value { bits | bit } else { bits & !bit };
4172 *a = mask16x8 {
4173 val: (bits) as _,
4174 simd: self,
4175 };
4176 }
4177 #[inline(always)]
4178 fn rotate_elements_left_mask16x8<const OFFSET: usize>(
4179 self,
4180 a: mask16x8<Self>,
4181 ) -> mask16x8<Self> {
4182 mask16x8 {
4183 val: (a.val.rotate_right((OFFSET % 8) as u32)) as _,
4184 simd: self,
4185 }
4186 }
4187 #[inline(always)]
4188 fn rotate_elements_right_mask16x8<const OFFSET: usize>(
4189 self,
4190 a: mask16x8<Self>,
4191 ) -> mask16x8<Self> {
4192 mask16x8 {
4193 val: (a.val.rotate_left((OFFSET % 8) as u32)) as _,
4194 simd: self,
4195 }
4196 }
4197 #[inline(always)]
4198 fn and_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
4199 mask16x8 {
4200 val: ((u64::from((a).val) & u64::from((b).val)) & 255u64) as _,
4201 simd: self,
4202 }
4203 }
4204 #[inline(always)]
4205 fn or_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
4206 mask16x8 {
4207 val: ((u64::from((a).val) | u64::from((b).val)) & 255u64) as _,
4208 simd: self,
4209 }
4210 }
4211 #[inline(always)]
4212 fn xor_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
4213 mask16x8 {
4214 val: ((u64::from((a).val) ^ u64::from((b).val)) & 255u64) as _,
4215 simd: self,
4216 }
4217 }
4218 #[inline(always)]
4219 fn not_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
4220 mask16x8 {
4221 val: ((!u64::from((a).val)) & 255u64) as _,
4222 simd: self,
4223 }
4224 }
4225 #[inline(always)]
4226 fn reverse_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
4227 mask16x8 {
4228 val: (a.val.reverse_bits() >> 0usize) as _,
4229 simd: self,
4230 }
4231 }
4232 #[inline(always)]
4233 fn select_mask16x8(
4234 self,
4235 a: mask16x8<Self>,
4236 b: mask16x8<Self>,
4237 c: mask16x8<Self>,
4238 ) -> mask16x8<Self> {
4239 mask16x8 {
4240 val: (((u64::from((a).val) & u64::from((b).val))
4241 | ((!u64::from((a).val)) & u64::from((c).val)))
4242 & 255u64) as _,
4243 simd: self,
4244 }
4245 }
4246 #[inline(always)]
4247 fn simd_eq_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
4248 mask16x8 {
4249 val: (!u64::from(a.val ^ b.val) & 255u64) as _,
4250 simd: self,
4251 }
4252 }
4253 #[inline(always)]
4254 fn simd_ne_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
4255 mask16x8 {
4256 val: (u64::from(a.val ^ b.val) & 255u64) as _,
4257 simd: self,
4258 }
4259 }
4260 #[inline(always)]
4261 fn any_true_mask16x8(self, a: mask16x8<Self>) -> bool {
4262 let bits = u64::from((a).val) & 255u64;
4263 bits != 0
4264 }
4265 #[inline(always)]
4266 fn all_true_mask16x8(self, a: mask16x8<Self>) -> bool {
4267 let bits = u64::from((a).val) & 255u64;
4268 bits == 255u64
4269 }
4270 #[inline(always)]
4271 fn any_false_mask16x8(self, a: mask16x8<Self>) -> bool {
4272 let bits = u64::from((a).val) & 255u64;
4273 bits != 255u64
4274 }
4275 #[inline(always)]
4276 fn all_false_mask16x8(self, a: mask16x8<Self>) -> bool {
4277 let bits = u64::from((a).val) & 255u64;
4278 bits == 0
4279 }
4280 #[inline(always)]
4281 fn combine_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x16<Self> {
4282 let bits = (u64::from(a.val) | (u64::from(b.val) << 8usize)) & 65535u64;
4283 mask16x16 {
4284 val: bits as _,
4285 simd: self,
4286 }
4287 }
4288 #[inline(always)]
4289 fn widen_mask16x8(self, a: mask16x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
4290 let bits = u64::from(a.val);
4291 (
4292 mask32x4 {
4293 val: (bits & 15u64) as _,
4294 simd: self,
4295 },
4296 mask32x4 {
4297 val: ((bits >> 4usize) & 15u64) as _,
4298 simd: self,
4299 },
4300 )
4301 }
4302 #[inline(always)]
4303 fn narrow_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask8x16<Self> {
4304 let low = u64::from(a.val) & 255u64;
4305 let high = u64::from(b.val) & 255u64;
4306 mask8x16 {
4307 val: (low | (high << 8usize)) as _,
4308 simd: self,
4309 }
4310 }
4311 #[inline(always)]
4312 fn abs_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4313 crate::kernel!(
4314 #[inline(always)]
4315 fn kernel(token: Avx512, a: i32x4<Avx512>) -> i32x4<Avx512> {
4316 _mm_abs_epi32(a.into()).simd_into(token)
4317 }
4318 );
4319 kernel(self, a)
4320 }
4321 #[inline(always)]
4322 fn splat_i32x4(self, val: i32) -> i32x4<Self> {
4323 crate::kernel!(
4324 #[inline(always)]
4325 fn kernel(token: Avx512, val: i32) -> i32x4<Avx512> {
4326 _mm_set1_epi32(val).simd_into(token)
4327 }
4328 );
4329 kernel(self, val)
4330 }
4331 #[inline(always)]
4332 fn reverse_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4333 let indices: u8x16<Self> =
4334 [12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3].simd_into(self);
4335 self.swizzle_dyn_i32x4(a, indices)
4336 }
4337 #[inline(always)]
4338 fn slide_i32x4<const SHIFT: usize>(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4339 if SHIFT >= 4usize {
4340 return b;
4341 }
4342 let result = dyn_alignr_128(
4343 self,
4344 Bytes::to_bytes(b).val.0,
4345 Bytes::to_bytes(a).val.0,
4346 SHIFT * 4usize,
4347 );
4348 Bytes::from_bytes(u8x16 {
4349 val: crate::support::Aligned128(result),
4350 simd: self,
4351 })
4352 }
4353 #[inline(always)]
4354 fn compress_i32x4(self, values: i32x4<Self>, mask: mask32x4<Self>) -> i32x4<Self> {
4355 crate::kernel!(
4356 #[inline(always)]
4357 fn kernel(
4358 token: Avx512,
4359 values: i32x4<Avx512>,
4360 mask: mask32x4<Avx512>,
4361 ) -> i32x4<Avx512> {
4362 _mm_maskz_compress_epi32(u64::from((mask).val) as u8, values.into())
4363 .simd_into(token)
4364 }
4365 );
4366 kernel(self, values, mask)
4367 }
4368 #[inline(always)]
4369 fn compress_merge_i32x4(
4370 self,
4371 values: i32x4<Self>,
4372 mask: mask32x4<Self>,
4373 merge: i32x4<Self>,
4374 ) -> i32x4<Self> {
4375 crate::kernel!(
4376 #[inline(always)]
4377 fn kernel(
4378 token: Avx512,
4379 values: i32x4<Avx512>,
4380 mask: mask32x4<Avx512>,
4381 merge: i32x4<Avx512>,
4382 ) -> i32x4<Avx512> {
4383 _mm_mask_compress_epi32(merge.into(), u64::from((mask).val) as u8, values.into())
4384 .simd_into(token)
4385 }
4386 );
4387 kernel(self, values, mask, merge)
4388 }
4389 #[inline(always)]
4390 fn expand_i32x4(self, values: i32x4<Self>, mask: mask32x4<Self>) -> i32x4<Self> {
4391 crate::kernel!(
4392 #[inline(always)]
4393 fn kernel(
4394 token: Avx512,
4395 values: i32x4<Avx512>,
4396 mask: mask32x4<Avx512>,
4397 ) -> i32x4<Avx512> {
4398 _mm_maskz_expand_epi32(u64::from((mask).val) as u8, values.into()).simd_into(token)
4399 }
4400 );
4401 kernel(self, values, mask)
4402 }
4403 #[inline(always)]
4404 fn expand_merge_i32x4(
4405 self,
4406 values: i32x4<Self>,
4407 mask: mask32x4<Self>,
4408 merge: i32x4<Self>,
4409 ) -> i32x4<Self> {
4410 crate::kernel!(
4411 #[inline(always)]
4412 fn kernel(
4413 token: Avx512,
4414 values: i32x4<Avx512>,
4415 mask: mask32x4<Avx512>,
4416 merge: i32x4<Avx512>,
4417 ) -> i32x4<Avx512> {
4418 _mm_mask_expand_epi32(merge.into(), u64::from((mask).val) as u8, values.into())
4419 .simd_into(token)
4420 }
4421 );
4422 kernel(self, values, mask, merge)
4423 }
4424 #[inline(always)]
4425 fn count_ones_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4426 crate::kernel!(
4427 #[inline(always)]
4428 fn kernel(token: Avx512, a: i32x4<Avx512>) -> i32x4<Avx512> {
4429 _mm_popcnt_epi32(a.into()).simd_into(token)
4430 }
4431 );
4432 kernel(self, a)
4433 }
4434 #[inline(always)]
4435 fn count_zeros_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4436 self.count_ones_i32x4(self.not_i32x4(a))
4437 }
4438 #[inline(always)]
4439 fn add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4440 crate::kernel!(
4441 #[inline(always)]
4442 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4443 _mm_add_epi32(a.into(), b.into()).simd_into(token)
4444 }
4445 );
4446 kernel(self, a, b)
4447 }
4448 #[inline(always)]
4449 fn saturating_add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4450 crate::kernel!(
4451 #[inline(always)]
4452 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4453 let a = a.into();
4454 let b = b.into();
4455 let wrapped = _mm_add_epi32(a, b);
4456 let overflow_bits = _mm_ternarylogic_epi32::<0x42>(a, b, wrapped);
4457 let overflow_mask = _mm_srai_epi32::<31>(overflow_bits);
4458 let direction = _mm_add_epi32(_mm_srli_epi32::<31>(a), _mm_set1_epi32(i32::MAX));
4459 _mm_ternarylogic_epi32::<0xca>(overflow_mask, direction, wrapped).simd_into(token)
4460 }
4461 );
4462 kernel(self, a, b)
4463 }
4464 #[inline(always)]
4465 fn sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4466 crate::kernel!(
4467 #[inline(always)]
4468 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4469 _mm_sub_epi32(a.into(), b.into()).simd_into(token)
4470 }
4471 );
4472 kernel(self, a, b)
4473 }
4474 #[inline(always)]
4475 fn saturating_sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4476 crate::kernel!(
4477 #[inline(always)]
4478 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4479 let a = a.into();
4480 let b = b.into();
4481 let wrapped = _mm_sub_epi32(a, b);
4482 let overflow_bits = _mm_ternarylogic_epi32::<0x18>(a, b, wrapped);
4483 let overflow_mask = _mm_srai_epi32::<31>(overflow_bits);
4484 let direction = _mm_add_epi32(_mm_srli_epi32::<31>(a), _mm_set1_epi32(i32::MAX));
4485 _mm_ternarylogic_epi32::<0xca>(overflow_mask, direction, wrapped).simd_into(token)
4486 }
4487 );
4488 kernel(self, a, b)
4489 }
4490 #[inline(always)]
4491 fn mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4492 crate::kernel!(
4493 #[inline(always)]
4494 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4495 _mm_mullo_epi32(a.into(), b.into()).simd_into(token)
4496 }
4497 );
4498 kernel(self, a, b)
4499 }
4500 #[inline(always)]
4501 fn and_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4502 crate::kernel!(
4503 #[inline(always)]
4504 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4505 _mm_and_si128(a.into(), b.into()).simd_into(token)
4506 }
4507 );
4508 kernel(self, a, b)
4509 }
4510 #[inline(always)]
4511 fn or_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4512 crate::kernel!(
4513 #[inline(always)]
4514 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4515 _mm_or_si128(a.into(), b.into()).simd_into(token)
4516 }
4517 );
4518 kernel(self, a, b)
4519 }
4520 #[inline(always)]
4521 fn xor_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4522 crate::kernel!(
4523 #[inline(always)]
4524 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4525 _mm_xor_si128(a.into(), b.into()).simd_into(token)
4526 }
4527 );
4528 kernel(self, a, b)
4529 }
4530 #[inline(always)]
4531 fn not_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4532 a ^ !0
4533 }
4534 #[inline(always)]
4535 fn shl_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
4536 crate::kernel!(
4537 #[inline(always)]
4538 fn kernel(token: Avx512, a: i32x4<Avx512>, shift: u32) -> i32x4<Avx512> {
4539 _mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
4540 }
4541 );
4542 kernel(self, a, shift)
4543 }
4544 #[inline(always)]
4545 fn shlv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4546 crate::kernel!(
4547 #[inline(always)]
4548 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4549 _mm_sllv_epi32(a.into(), b.into()).simd_into(token)
4550 }
4551 );
4552 kernel(self, a, b)
4553 }
4554 #[inline(always)]
4555 fn shr_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
4556 crate::kernel!(
4557 #[inline(always)]
4558 fn kernel(token: Avx512, a: i32x4<Avx512>, shift: u32) -> i32x4<Avx512> {
4559 _mm_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
4560 }
4561 );
4562 kernel(self, a, shift)
4563 }
4564 #[inline(always)]
4565 fn shrv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4566 crate::kernel!(
4567 #[inline(always)]
4568 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4569 _mm_srav_epi32(a.into(), b.into()).simd_into(token)
4570 }
4571 );
4572 kernel(self, a, b)
4573 }
4574 #[inline(always)]
4575 fn reduce_max_i32x4(self, a: i32x4<Self>) -> i32 {
4576 crate::kernel!(
4577 #[inline(always)]
4578 fn kernel(token: Avx512, a: i32x4<Avx512>) -> i32 {
4579 let reduced: __m128i = a.into();
4580 let shifted = _mm_srli_si128::<8>(reduced);
4581 let reduced = { _mm_max_epi32(reduced, shifted) };
4582 let shifted = _mm_srli_si128::<4>(reduced);
4583 let reduced = { _mm_max_epi32(reduced, shifted) };
4584 {
4585 let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
4586 lanes[0]
4587 }
4588 }
4589 );
4590 kernel(self, a)
4591 }
4592 #[inline(always)]
4593 fn reduce_min_i32x4(self, a: i32x4<Self>) -> i32 {
4594 crate::kernel!(
4595 #[inline(always)]
4596 fn kernel(token: Avx512, a: i32x4<Avx512>) -> i32 {
4597 let reduced: __m128i = a.into();
4598 let shifted = _mm_srli_si128::<8>(reduced);
4599 let reduced = { _mm_min_epi32(reduced, shifted) };
4600 let shifted = _mm_srli_si128::<4>(reduced);
4601 let reduced = { _mm_min_epi32(reduced, shifted) };
4602 {
4603 let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
4604 lanes[0]
4605 }
4606 }
4607 );
4608 kernel(self, a)
4609 }
4610 #[inline(always)]
4611 fn reduce_sum_i32x4(self, a: i32x4<Self>) -> i32 {
4612 crate::kernel!(
4613 #[inline(always)]
4614 fn kernel(token: Avx512, a: i32x4<Avx512>) -> i32 {
4615 let sum: __m128i = a.into();
4616 let sum = _mm_add_epi32(sum, _mm_srli_si128::<8>(sum));
4617 let sum = _mm_add_epi32(sum, _mm_srli_si128::<4>(sum));
4618 let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&sum);
4619 lanes[0]
4620 }
4621 );
4622 kernel(self, a)
4623 }
4624 #[inline(always)]
4625 fn reduce_product_i32x4(self, a: i32x4<Self>) -> i32 {
4626 let mul_level_0: [i32; 2usize] = [
4627 a[0usize].wrapping_mul(a[1usize]),
4628 a[2usize].wrapping_mul(a[3usize]),
4629 ];
4630 let mul_level_1: [i32; 1usize] = [mul_level_0[0usize].wrapping_mul(mul_level_0[1usize])];
4631 mul_level_1[0]
4632 }
4633 #[inline(always)]
4634 fn max_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4635 crate::kernel!(
4636 #[inline(always)]
4637 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4638 _mm_max_epi32(a.into(), b.into()).simd_into(token)
4639 }
4640 );
4641 kernel(self, a, b)
4642 }
4643 #[inline(always)]
4644 fn min_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4645 crate::kernel!(
4646 #[inline(always)]
4647 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4648 _mm_min_epi32(a.into(), b.into()).simd_into(token)
4649 }
4650 );
4651 kernel(self, a, b)
4652 }
4653 #[inline(always)]
4654 fn clamp_i32x4(self, a: i32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
4655 debug_assert!(
4656 b.simd_le(c).all_true(),
4657 "clamp requires min <= max for all elements"
4658 );
4659 a.max(b).min(c)
4660 }
4661 #[inline(always)]
4662 fn simd_eq_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
4663 crate::kernel!(
4664 #[inline(always)]
4665 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> mask32x4<Avx512> {
4666 mask32x4 {
4667 val: _mm_cmpeq_epi32_mask(a.into(), b.into()),
4668 simd: token,
4669 }
4670 }
4671 );
4672 kernel(self, a, b)
4673 }
4674 #[inline(always)]
4675 fn simd_ne_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
4676 crate::kernel!(
4677 #[inline(always)]
4678 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> mask32x4<Avx512> {
4679 mask32x4 {
4680 val: _mm_cmpneq_epi32_mask(a.into(), b.into()),
4681 simd: token,
4682 }
4683 }
4684 );
4685 kernel(self, a, b)
4686 }
4687 #[inline(always)]
4688 fn simd_lt_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
4689 crate::kernel!(
4690 #[inline(always)]
4691 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> mask32x4<Avx512> {
4692 mask32x4 {
4693 val: _mm_cmplt_epi32_mask(a.into(), b.into()),
4694 simd: token,
4695 }
4696 }
4697 );
4698 kernel(self, a, b)
4699 }
4700 #[inline(always)]
4701 fn simd_le_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
4702 crate::kernel!(
4703 #[inline(always)]
4704 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> mask32x4<Avx512> {
4705 mask32x4 {
4706 val: _mm_cmple_epi32_mask(a.into(), b.into()),
4707 simd: token,
4708 }
4709 }
4710 );
4711 kernel(self, a, b)
4712 }
4713 #[inline(always)]
4714 fn zip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4715 crate::kernel!(
4716 #[inline(always)]
4717 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4718 _mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
4719 }
4720 );
4721 kernel(self, a, b)
4722 }
4723 #[inline(always)]
4724 fn zip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4725 crate::kernel!(
4726 #[inline(always)]
4727 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4728 _mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
4729 }
4730 );
4731 kernel(self, a, b)
4732 }
4733 #[inline(always)]
4734 fn unzip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4735 crate::kernel!(
4736 #[inline(always)]
4737 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4738 _mm_permutex2var_epi32(a.into(), _mm_setr_epi32(0, 2, 4, 6), b.into())
4739 .simd_into(token)
4740 }
4741 );
4742 kernel(self, a, b)
4743 }
4744 #[inline(always)]
4745 fn unzip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4746 crate::kernel!(
4747 #[inline(always)]
4748 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
4749 _mm_permutex2var_epi32(a.into(), _mm_setr_epi32(1, 3, 5, 7), b.into())
4750 .simd_into(token)
4751 }
4752 );
4753 kernel(self, a, b)
4754 }
4755 #[inline(always)]
4756 fn interleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
4757 (self.zip_low_i32x4(a, b), self.zip_high_i32x4(a, b))
4758 }
4759 #[inline(always)]
4760 fn deinterleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
4761 crate::kernel!(
4762 #[inline(always)]
4763 fn kernel(
4764 token: Avx512,
4765 a: i32x4<Avx512>,
4766 b: i32x4<Avx512>,
4767 ) -> (i32x4<Avx512>, i32x4<Avx512>) {
4768 let a = a.into();
4769 let b = b.into();
4770 (
4771 _mm_permutex2var_epi32(a, _mm_setr_epi32(0, 2, 4, 6), b).simd_into(token),
4772 _mm_permutex2var_epi32(a, _mm_setr_epi32(1, 3, 5, 7), b).simd_into(token),
4773 )
4774 }
4775 );
4776 kernel(self, a, b)
4777 }
4778 #[inline(always)]
4779 fn select_i32x4(self, a: mask32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
4780 crate::kernel!(
4781 #[inline(always)]
4782 fn kernel(
4783 token: Avx512,
4784 a: mask32x4<Avx512>,
4785 b: i32x4<Avx512>,
4786 c: i32x4<Avx512>,
4787 ) -> i32x4<Avx512> {
4788 _mm_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token)
4789 }
4790 );
4791 kernel(self, a, b, c)
4792 }
4793 #[inline(always)]
4794 fn combine_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x8<Self> {
4795 crate::kernel!(
4796 #[inline(always)]
4797 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x8<Avx512> {
4798 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
4799 }
4800 );
4801 kernel(self, a, b)
4802 }
4803 #[inline(always)]
4804 fn neg_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4805 crate::kernel!(
4806 #[inline(always)]
4807 fn kernel(token: Avx512, a: i32x4<Avx512>) -> i32x4<Avx512> {
4808 _mm_sub_epi32(_mm_setzero_si128(), a.into()).simd_into(token)
4809 }
4810 );
4811 kernel(self, a)
4812 }
4813 #[inline(always)]
4814 fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4<Self>; 4usize] {
4815 crate::kernel!(
4816 #[inline(always)]
4817 fn kernel(token: Avx512, src: &[i32; 16usize]) -> [i32x4<Avx512>; 4usize] {
4818 let lanes: __m512i =
4819 crate::transmute::checked_transmute_copy::<[i32; 16usize], __m512i>(src);
4820 let lanes = _mm512_permutexvar_epi32(
4821 _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15),
4822 lanes,
4823 );
4824 let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes);
4825 [
4826 outputs[0].simd_into(token),
4827 outputs[1].simd_into(token),
4828 outputs[2].simd_into(token),
4829 outputs[3].simd_into(token),
4830 ]
4831 }
4832 );
4833 kernel(self, src)
4834 }
4835 #[inline(always)]
4836 fn store_four_interleaved_i32x4(
4837 self,
4838 vectors: [i32x4<Self>; 4usize],
4839 dest: &mut [i32; 16usize],
4840 ) -> () {
4841 crate::kernel!(
4842 #[inline(always)]
4843 fn kernel(
4844 token: Avx512,
4845 vectors: [i32x4<Avx512>; 4usize],
4846 dest: &mut [i32; 16usize],
4847 ) -> () {
4848 let _ = token;
4849 let inputs: [__m128i; 4] = [
4850 vectors[0].into(),
4851 vectors[1].into(),
4852 vectors[2].into(),
4853 vectors[3].into(),
4854 ];
4855 let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs);
4856 let lanes = _mm512_permutexvar_epi32(
4857 _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15),
4858 lanes,
4859 );
4860 crate::transmute::checked_transmute_store::<__m512i, [i32; 16usize]>(lanes, dest);
4861 }
4862 );
4863 kernel(self, vectors, dest);
4864 }
4865 #[inline(always)]
4866 fn widen_i32x4(self, a: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
4867 crate::kernel!(
4868 #[inline(always)]
4869 fn kernel(token: Avx512, a: i32x4<Avx512>) -> (i64x2<Avx512>, i64x2<Avx512>) {
4870 let raw = a.into();
4871 (
4872 _mm_cvtepi32_epi64(raw).simd_into(token),
4873 _mm_cvtepi32_epi64(_mm_srli_si128::<8>(raw)).simd_into(token),
4874 )
4875 }
4876 );
4877 kernel(self, a)
4878 }
4879 #[inline(always)]
4880 fn widening_mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
4881 let (a0, a1) = self.widen_i32x4(a);
4882 let (b0, b1) = self.widen_i32x4(b);
4883 (self.mul_i64x2(a0, b0), self.mul_i64x2(a1, b1))
4884 }
4885 #[inline(always)]
4886 fn widening_add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
4887 let (a0, a1) = self.widen_i32x4(a);
4888 let (b0, b1) = self.widen_i32x4(b);
4889 (self.add_i64x2(a0, b0), self.add_i64x2(a1, b1))
4890 }
4891 #[inline(always)]
4892 fn pairwise_widening_add_i32x4(self, a: i32x4<Self>) -> i64x2<Self> {
4893 crate::kernel!(
4894 #[inline(always)]
4895 fn kernel(token: Avx512, a: i32x4<Avx512>) -> i64x2<Avx512> {
4896 ({
4897 let y = _mm_xor_si128(a.into(), _mm_set1_epi32(i32::MIN));
4898 _mm_add_epi64(
4899 _mm_or_si128(y, _mm_set1_epi64x(-0x1_0000_0000)),
4900 _mm_srli_epi64::<32>(y),
4901 )
4902 })
4903 .simd_into(token)
4904 }
4905 );
4906 kernel(self, a)
4907 }
4908 #[inline(always)]
4909 fn pairwise_dot_product_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i64x2<Self> {
4910 let (low, high) = self.widening_mul_i32x4(a, b);
4911 let (even, odd) = self.deinterleave_i64x2(low, high);
4912 self.add_i64x2(even, odd)
4913 }
4914 #[inline(always)]
4915 fn narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
4916 crate::kernel!(
4917 #[inline(always)]
4918 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i16x8<Avx512> {
4919 let low = _mm_cvtepi32_epi16(a.into());
4920 let high = _mm_cvtepi32_epi16(b.into());
4921 _mm_unpacklo_epi64(low, high).simd_into(token)
4922 }
4923 );
4924 kernel(self, a, b)
4925 }
4926 #[inline(always)]
4927 fn saturating_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
4928 crate::kernel!(
4929 #[inline(always)]
4930 fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i16x8<Avx512> {
4931 _mm_packs_epi32(a.into(), b.into()).simd_into(token)
4932 }
4933 );
4934 kernel(self, a, b)
4935 }
4936 #[inline(always)]
4937 fn relaxed_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
4938 debug_assert!(
4939 a.as_slice()
4940 .iter()
4941 .chain(b.as_slice())
4942 .all(|&value| { value >= i16::MIN as i32 && value <= i16::MAX as i32 }),
4943 "relaxed_narrow inputs must fit in the destination type",
4944 );
4945 self.saturating_narrow_i32x4(a, b)
4946 }
4947 #[inline(always)]
4948 fn cvt_f32_i32x4(self, a: i32x4<Self>) -> f32x4<Self> {
4949 crate::kernel!(
4950 #[inline(always)]
4951 fn kernel(token: Avx512, a: i32x4<Avx512>) -> f32x4<Avx512> {
4952 _mm_cvtepi32_ps(a.into()).simd_into(token)
4953 }
4954 );
4955 kernel(self, a)
4956 }
4957 #[inline(always)]
4958 fn splat_u32x4(self, val: u32) -> u32x4<Self> {
4959 crate::kernel!(
4960 #[inline(always)]
4961 fn kernel(token: Avx512, val: u32) -> u32x4<Avx512> {
4962 _mm_set1_epi32(val.cast_signed()).simd_into(token)
4963 }
4964 );
4965 kernel(self, val)
4966 }
4967 #[inline(always)]
4968 fn reverse_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
4969 let indices: u8x16<Self> =
4970 [12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3].simd_into(self);
4971 self.swizzle_dyn_u32x4(a, indices)
4972 }
4973 #[inline(always)]
4974 fn slide_u32x4<const SHIFT: usize>(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4975 if SHIFT >= 4usize {
4976 return b;
4977 }
4978 let result = dyn_alignr_128(
4979 self,
4980 Bytes::to_bytes(b).val.0,
4981 Bytes::to_bytes(a).val.0,
4982 SHIFT * 4usize,
4983 );
4984 Bytes::from_bytes(u8x16 {
4985 val: crate::support::Aligned128(result),
4986 simd: self,
4987 })
4988 }
4989 #[inline(always)]
4990 fn compress_u32x4(self, values: u32x4<Self>, mask: mask32x4<Self>) -> u32x4<Self> {
4991 crate::kernel!(
4992 #[inline(always)]
4993 fn kernel(
4994 token: Avx512,
4995 values: u32x4<Avx512>,
4996 mask: mask32x4<Avx512>,
4997 ) -> u32x4<Avx512> {
4998 _mm_maskz_compress_epi32(u64::from((mask).val) as u8, values.into())
4999 .simd_into(token)
5000 }
5001 );
5002 kernel(self, values, mask)
5003 }
5004 #[inline(always)]
5005 fn compress_merge_u32x4(
5006 self,
5007 values: u32x4<Self>,
5008 mask: mask32x4<Self>,
5009 merge: u32x4<Self>,
5010 ) -> u32x4<Self> {
5011 crate::kernel!(
5012 #[inline(always)]
5013 fn kernel(
5014 token: Avx512,
5015 values: u32x4<Avx512>,
5016 mask: mask32x4<Avx512>,
5017 merge: u32x4<Avx512>,
5018 ) -> u32x4<Avx512> {
5019 _mm_mask_compress_epi32(merge.into(), u64::from((mask).val) as u8, values.into())
5020 .simd_into(token)
5021 }
5022 );
5023 kernel(self, values, mask, merge)
5024 }
5025 #[inline(always)]
5026 fn expand_u32x4(self, values: u32x4<Self>, mask: mask32x4<Self>) -> u32x4<Self> {
5027 crate::kernel!(
5028 #[inline(always)]
5029 fn kernel(
5030 token: Avx512,
5031 values: u32x4<Avx512>,
5032 mask: mask32x4<Avx512>,
5033 ) -> u32x4<Avx512> {
5034 _mm_maskz_expand_epi32(u64::from((mask).val) as u8, values.into()).simd_into(token)
5035 }
5036 );
5037 kernel(self, values, mask)
5038 }
5039 #[inline(always)]
5040 fn expand_merge_u32x4(
5041 self,
5042 values: u32x4<Self>,
5043 mask: mask32x4<Self>,
5044 merge: u32x4<Self>,
5045 ) -> u32x4<Self> {
5046 crate::kernel!(
5047 #[inline(always)]
5048 fn kernel(
5049 token: Avx512,
5050 values: u32x4<Avx512>,
5051 mask: mask32x4<Avx512>,
5052 merge: u32x4<Avx512>,
5053 ) -> u32x4<Avx512> {
5054 _mm_mask_expand_epi32(merge.into(), u64::from((mask).val) as u8, values.into())
5055 .simd_into(token)
5056 }
5057 );
5058 kernel(self, values, mask, merge)
5059 }
5060 #[inline(always)]
5061 fn count_ones_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
5062 crate::kernel!(
5063 #[inline(always)]
5064 fn kernel(token: Avx512, a: u32x4<Avx512>) -> u32x4<Avx512> {
5065 _mm_popcnt_epi32(a.into()).simd_into(token)
5066 }
5067 );
5068 kernel(self, a)
5069 }
5070 #[inline(always)]
5071 fn count_zeros_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
5072 self.count_ones_u32x4(self.not_u32x4(a))
5073 }
5074 #[inline(always)]
5075 fn add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5076 crate::kernel!(
5077 #[inline(always)]
5078 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5079 _mm_add_epi32(a.into(), b.into()).simd_into(token)
5080 }
5081 );
5082 kernel(self, a, b)
5083 }
5084 #[inline(always)]
5085 fn saturating_add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5086 crate::kernel!(
5087 #[inline(always)]
5088 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5089 let a = a.into();
5090 let b = b.into();
5091 let threshold = _mm_xor_si128(b, _mm_set1_epi32(-1));
5092 _mm_add_epi32(_mm_min_epu32(a, threshold), b).simd_into(token)
5093 }
5094 );
5095 kernel(self, a, b)
5096 }
5097 #[inline(always)]
5098 fn sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5099 crate::kernel!(
5100 #[inline(always)]
5101 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5102 _mm_sub_epi32(a.into(), b.into()).simd_into(token)
5103 }
5104 );
5105 kernel(self, a, b)
5106 }
5107 #[inline(always)]
5108 fn saturating_sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5109 crate::kernel!(
5110 #[inline(always)]
5111 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5112 let a = a.into();
5113 let b = b.into();
5114 _mm_sub_epi32(_mm_max_epu32(a, b), b).simd_into(token)
5115 }
5116 );
5117 kernel(self, a, b)
5118 }
5119 #[inline(always)]
5120 fn mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5121 crate::kernel!(
5122 #[inline(always)]
5123 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5124 _mm_mullo_epi32(a.into(), b.into()).simd_into(token)
5125 }
5126 );
5127 kernel(self, a, b)
5128 }
5129 #[inline(always)]
5130 fn and_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5131 crate::kernel!(
5132 #[inline(always)]
5133 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5134 _mm_and_si128(a.into(), b.into()).simd_into(token)
5135 }
5136 );
5137 kernel(self, a, b)
5138 }
5139 #[inline(always)]
5140 fn or_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5141 crate::kernel!(
5142 #[inline(always)]
5143 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5144 _mm_or_si128(a.into(), b.into()).simd_into(token)
5145 }
5146 );
5147 kernel(self, a, b)
5148 }
5149 #[inline(always)]
5150 fn xor_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5151 crate::kernel!(
5152 #[inline(always)]
5153 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5154 _mm_xor_si128(a.into(), b.into()).simd_into(token)
5155 }
5156 );
5157 kernel(self, a, b)
5158 }
5159 #[inline(always)]
5160 fn not_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
5161 a ^ !0
5162 }
5163 #[inline(always)]
5164 fn shl_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
5165 crate::kernel!(
5166 #[inline(always)]
5167 fn kernel(token: Avx512, a: u32x4<Avx512>, shift: u32) -> u32x4<Avx512> {
5168 _mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
5169 }
5170 );
5171 kernel(self, a, shift)
5172 }
5173 #[inline(always)]
5174 fn shlv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5175 crate::kernel!(
5176 #[inline(always)]
5177 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5178 _mm_sllv_epi32(a.into(), b.into()).simd_into(token)
5179 }
5180 );
5181 kernel(self, a, b)
5182 }
5183 #[inline(always)]
5184 fn shr_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
5185 crate::kernel!(
5186 #[inline(always)]
5187 fn kernel(token: Avx512, a: u32x4<Avx512>, shift: u32) -> u32x4<Avx512> {
5188 _mm_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
5189 }
5190 );
5191 kernel(self, a, shift)
5192 }
5193 #[inline(always)]
5194 fn shrv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5195 crate::kernel!(
5196 #[inline(always)]
5197 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5198 _mm_srlv_epi32(a.into(), b.into()).simd_into(token)
5199 }
5200 );
5201 kernel(self, a, b)
5202 }
5203 #[inline(always)]
5204 fn reduce_max_u32x4(self, a: u32x4<Self>) -> u32 {
5205 crate::kernel!(
5206 #[inline(always)]
5207 fn kernel(token: Avx512, a: u32x4<Avx512>) -> u32 {
5208 let reduced: __m128i = a.into();
5209 let shifted = _mm_srli_si128::<8>(reduced);
5210 let reduced = { _mm_max_epu32(reduced, shifted) };
5211 let shifted = _mm_srli_si128::<4>(reduced);
5212 let reduced = { _mm_max_epu32(reduced, shifted) };
5213 {
5214 let lanes: [u32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
5215 lanes[0]
5216 }
5217 }
5218 );
5219 kernel(self, a)
5220 }
5221 #[inline(always)]
5222 fn reduce_min_u32x4(self, a: u32x4<Self>) -> u32 {
5223 crate::kernel!(
5224 #[inline(always)]
5225 fn kernel(token: Avx512, a: u32x4<Avx512>) -> u32 {
5226 let reduced: __m128i = a.into();
5227 let shifted = _mm_srli_si128::<8>(reduced);
5228 let reduced = { _mm_min_epu32(reduced, shifted) };
5229 let shifted = _mm_srli_si128::<4>(reduced);
5230 let reduced = { _mm_min_epu32(reduced, shifted) };
5231 {
5232 let lanes: [u32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
5233 lanes[0]
5234 }
5235 }
5236 );
5237 kernel(self, a)
5238 }
5239 #[inline(always)]
5240 fn reduce_sum_u32x4(self, a: u32x4<Self>) -> u32 {
5241 crate::kernel!(
5242 #[inline(always)]
5243 fn kernel(token: Avx512, a: u32x4<Avx512>) -> u32 {
5244 let sum: __m128i = a.into();
5245 let sum = _mm_add_epi32(sum, _mm_srli_si128::<8>(sum));
5246 let sum = _mm_add_epi32(sum, _mm_srli_si128::<4>(sum));
5247 let lanes: [u32; 4usize] = crate::transmute::checked_transmute_copy(&sum);
5248 lanes[0]
5249 }
5250 );
5251 kernel(self, a)
5252 }
5253 #[inline(always)]
5254 fn reduce_product_u32x4(self, a: u32x4<Self>) -> u32 {
5255 let mul_level_0: [u32; 2usize] = [
5256 a[0usize].wrapping_mul(a[1usize]),
5257 a[2usize].wrapping_mul(a[3usize]),
5258 ];
5259 let mul_level_1: [u32; 1usize] = [mul_level_0[0usize].wrapping_mul(mul_level_0[1usize])];
5260 mul_level_1[0]
5261 }
5262 #[inline(always)]
5263 fn max_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5264 crate::kernel!(
5265 #[inline(always)]
5266 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5267 _mm_max_epu32(a.into(), b.into()).simd_into(token)
5268 }
5269 );
5270 kernel(self, a, b)
5271 }
5272 #[inline(always)]
5273 fn min_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5274 crate::kernel!(
5275 #[inline(always)]
5276 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5277 _mm_min_epu32(a.into(), b.into()).simd_into(token)
5278 }
5279 );
5280 kernel(self, a, b)
5281 }
5282 #[inline(always)]
5283 fn clamp_u32x4(self, a: u32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
5284 debug_assert!(
5285 b.simd_le(c).all_true(),
5286 "clamp requires min <= max for all elements"
5287 );
5288 a.max(b).min(c)
5289 }
5290 #[inline(always)]
5291 fn simd_eq_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5292 crate::kernel!(
5293 #[inline(always)]
5294 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> mask32x4<Avx512> {
5295 mask32x4 {
5296 val: _mm_cmpeq_epu32_mask(a.into(), b.into()),
5297 simd: token,
5298 }
5299 }
5300 );
5301 kernel(self, a, b)
5302 }
5303 #[inline(always)]
5304 fn simd_ne_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5305 crate::kernel!(
5306 #[inline(always)]
5307 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> mask32x4<Avx512> {
5308 mask32x4 {
5309 val: _mm_cmpneq_epu32_mask(a.into(), b.into()),
5310 simd: token,
5311 }
5312 }
5313 );
5314 kernel(self, a, b)
5315 }
5316 #[inline(always)]
5317 fn simd_lt_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5318 crate::kernel!(
5319 #[inline(always)]
5320 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> mask32x4<Avx512> {
5321 mask32x4 {
5322 val: _mm_cmplt_epu32_mask(a.into(), b.into()),
5323 simd: token,
5324 }
5325 }
5326 );
5327 kernel(self, a, b)
5328 }
5329 #[inline(always)]
5330 fn simd_le_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5331 crate::kernel!(
5332 #[inline(always)]
5333 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> mask32x4<Avx512> {
5334 mask32x4 {
5335 val: _mm_cmple_epu32_mask(a.into(), b.into()),
5336 simd: token,
5337 }
5338 }
5339 );
5340 kernel(self, a, b)
5341 }
5342 #[inline(always)]
5343 fn zip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5344 crate::kernel!(
5345 #[inline(always)]
5346 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5347 _mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
5348 }
5349 );
5350 kernel(self, a, b)
5351 }
5352 #[inline(always)]
5353 fn zip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5354 crate::kernel!(
5355 #[inline(always)]
5356 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5357 _mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
5358 }
5359 );
5360 kernel(self, a, b)
5361 }
5362 #[inline(always)]
5363 fn unzip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5364 crate::kernel!(
5365 #[inline(always)]
5366 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5367 _mm_permutex2var_epi32(a.into(), _mm_setr_epi32(0, 2, 4, 6), b.into())
5368 .simd_into(token)
5369 }
5370 );
5371 kernel(self, a, b)
5372 }
5373 #[inline(always)]
5374 fn unzip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5375 crate::kernel!(
5376 #[inline(always)]
5377 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
5378 _mm_permutex2var_epi32(a.into(), _mm_setr_epi32(1, 3, 5, 7), b.into())
5379 .simd_into(token)
5380 }
5381 );
5382 kernel(self, a, b)
5383 }
5384 #[inline(always)]
5385 fn interleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
5386 (self.zip_low_u32x4(a, b), self.zip_high_u32x4(a, b))
5387 }
5388 #[inline(always)]
5389 fn deinterleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
5390 crate::kernel!(
5391 #[inline(always)]
5392 fn kernel(
5393 token: Avx512,
5394 a: u32x4<Avx512>,
5395 b: u32x4<Avx512>,
5396 ) -> (u32x4<Avx512>, u32x4<Avx512>) {
5397 let a = a.into();
5398 let b = b.into();
5399 (
5400 _mm_permutex2var_epi32(a, _mm_setr_epi32(0, 2, 4, 6), b).simd_into(token),
5401 _mm_permutex2var_epi32(a, _mm_setr_epi32(1, 3, 5, 7), b).simd_into(token),
5402 )
5403 }
5404 );
5405 kernel(self, a, b)
5406 }
5407 #[inline(always)]
5408 fn select_u32x4(self, a: mask32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
5409 crate::kernel!(
5410 #[inline(always)]
5411 fn kernel(
5412 token: Avx512,
5413 a: mask32x4<Avx512>,
5414 b: u32x4<Avx512>,
5415 c: u32x4<Avx512>,
5416 ) -> u32x4<Avx512> {
5417 _mm_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token)
5418 }
5419 );
5420 kernel(self, a, b, c)
5421 }
5422 #[inline(always)]
5423 fn combine_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x8<Self> {
5424 crate::kernel!(
5425 #[inline(always)]
5426 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x8<Avx512> {
5427 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
5428 }
5429 );
5430 kernel(self, a, b)
5431 }
5432 #[inline(always)]
5433 fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4<Self>; 4usize] {
5434 crate::kernel!(
5435 #[inline(always)]
5436 fn kernel(token: Avx512, src: &[u32; 16usize]) -> [u32x4<Avx512>; 4usize] {
5437 let lanes: __m512i =
5438 crate::transmute::checked_transmute_copy::<[u32; 16usize], __m512i>(src);
5439 let lanes = _mm512_permutexvar_epi32(
5440 _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15),
5441 lanes,
5442 );
5443 let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes);
5444 [
5445 outputs[0].simd_into(token),
5446 outputs[1].simd_into(token),
5447 outputs[2].simd_into(token),
5448 outputs[3].simd_into(token),
5449 ]
5450 }
5451 );
5452 kernel(self, src)
5453 }
5454 #[inline(always)]
5455 fn store_four_interleaved_u32x4(
5456 self,
5457 vectors: [u32x4<Self>; 4usize],
5458 dest: &mut [u32; 16usize],
5459 ) -> () {
5460 crate::kernel!(
5461 #[inline(always)]
5462 fn kernel(
5463 token: Avx512,
5464 vectors: [u32x4<Avx512>; 4usize],
5465 dest: &mut [u32; 16usize],
5466 ) -> () {
5467 let _ = token;
5468 let inputs: [__m128i; 4] = [
5469 vectors[0].into(),
5470 vectors[1].into(),
5471 vectors[2].into(),
5472 vectors[3].into(),
5473 ];
5474 let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs);
5475 let lanes = _mm512_permutexvar_epi32(
5476 _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15),
5477 lanes,
5478 );
5479 crate::transmute::checked_transmute_store::<__m512i, [u32; 16usize]>(lanes, dest);
5480 }
5481 );
5482 kernel(self, vectors, dest);
5483 }
5484 #[inline(always)]
5485 fn widen_u32x4(self, a: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5486 crate::kernel!(
5487 #[inline(always)]
5488 fn kernel(token: Avx512, a: u32x4<Avx512>) -> (u64x2<Avx512>, u64x2<Avx512>) {
5489 let raw = a.into();
5490 (
5491 _mm_cvtepu32_epi64(raw).simd_into(token),
5492 _mm_cvtepu32_epi64(_mm_srli_si128::<8>(raw)).simd_into(token),
5493 )
5494 }
5495 );
5496 kernel(self, a)
5497 }
5498 #[inline(always)]
5499 fn widening_mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5500 let (a0, a1) = self.widen_u32x4(a);
5501 let (b0, b1) = self.widen_u32x4(b);
5502 (self.mul_u64x2(a0, b0), self.mul_u64x2(a1, b1))
5503 }
5504 #[inline(always)]
5505 fn widening_add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5506 let (a0, a1) = self.widen_u32x4(a);
5507 let (b0, b1) = self.widen_u32x4(b);
5508 (self.add_u64x2(a0, b0), self.add_u64x2(a1, b1))
5509 }
5510 #[inline(always)]
5511 fn pairwise_widening_add_u32x4(self, a: u32x4<Self>) -> u64x2<Self> {
5512 crate::kernel!(
5513 #[inline(always)]
5514 fn kernel(token: Avx512, a: u32x4<Avx512>) -> u64x2<Avx512> {
5515 ({
5516 let a = a.into();
5517 _mm_add_epi64(
5518 _mm_and_si128(a, _mm_set1_epi64x(4294967295)),
5519 _mm_srli_epi64::<32>(a),
5520 )
5521 })
5522 .simd_into(token)
5523 }
5524 );
5525 kernel(self, a)
5526 }
5527 #[inline(always)]
5528 fn pairwise_dot_product_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u64x2<Self> {
5529 let (low, high) = self.widening_mul_u32x4(a, b);
5530 let (even, odd) = self.deinterleave_u64x2(low, high);
5531 self.add_u64x2(even, odd)
5532 }
5533 #[inline(always)]
5534 fn narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5535 crate::kernel!(
5536 #[inline(always)]
5537 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u16x8<Avx512> {
5538 let low = _mm_cvtepi32_epi16(a.into());
5539 let high = _mm_cvtepi32_epi16(b.into());
5540 _mm_unpacklo_epi64(low, high).simd_into(token)
5541 }
5542 );
5543 kernel(self, a, b)
5544 }
5545 #[inline(always)]
5546 fn saturating_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5547 crate::kernel!(
5548 #[inline(always)]
5549 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u16x8<Avx512> {
5550 {
5551 let max = _mm_set1_epi32(i32::MAX);
5552 _mm_packus_epi32(_mm_min_epu32(a.into(), max), _mm_min_epu32(b.into(), max))
5553 }
5554 .simd_into(token)
5555 }
5556 );
5557 kernel(self, a, b)
5558 }
5559 #[inline(always)]
5560 fn relaxed_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5561 crate::kernel!(
5562 #[inline(always)]
5563 fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u16x8<Avx512> {
5564 debug_assert!(
5565 a.as_slice()
5566 .iter()
5567 .chain(b.as_slice())
5568 .all(|&value| { value >= u16::MIN as u32 && value <= u16::MAX as u32 }),
5569 "relaxed_narrow inputs must fit in the destination type",
5570 );
5571 _mm_packus_epi32(a.into(), b.into()).simd_into(token)
5572 }
5573 );
5574 kernel(self, a, b)
5575 }
5576 #[inline(always)]
5577 fn cvt_f32_u32x4(self, a: u32x4<Self>) -> f32x4<Self> {
5578 crate::kernel!(
5579 #[inline(always)]
5580 fn kernel(token: Avx512, a: u32x4<Avx512>) -> f32x4<Avx512> {
5581 _mm512_castps512_ps128(_mm512_cvtepu32_ps(_mm512_zextsi128_si512(a.into())))
5582 .simd_into(token)
5583 }
5584 );
5585 kernel(self, a)
5586 }
5587 #[inline(always)]
5588 fn splat_mask32x4(self, val: bool) -> mask32x4<Self> {
5589 mask32x4 {
5590 val: (if val { 15u64 } else { 0 }) as _,
5591 simd: self,
5592 }
5593 }
5594 #[inline(always)]
5595 fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4<Self> {
5596 mask32x4 {
5597 val: (bits & 15u64) as _,
5598 simd: self,
5599 }
5600 }
5601 #[inline(always)]
5602 fn to_bitmask_mask32x4(self, a: mask32x4<Self>) -> u64 {
5603 u64::from((a).val) & 15u64
5604 }
5605 #[inline(always)]
5606 fn from_vector_mask32x4(self, vector: i32x4<Self>) -> mask32x4<Self> {
5607 crate::kernel!(
5608 #[inline(always)]
5609 fn kernel(token: Avx512, vector: i32x4<Avx512>) -> mask32x4<Avx512> {
5610 _mm_movepi32_mask(vector.into()).simd_into(token)
5611 }
5612 );
5613 kernel(self, vector)
5614 }
5615 #[inline(always)]
5616 fn to_vector_mask32x4(self, a: mask32x4<Self>) -> i32x4<Self> {
5617 crate::kernel!(
5618 #[inline(always)]
5619 fn kernel(token: Avx512, a: mask32x4<Avx512>) -> i32x4<Avx512> {
5620 _mm_movm_epi32(a.val).simd_into(token)
5621 }
5622 );
5623 kernel(self, a)
5624 }
5625 #[inline(always)]
5626 fn set_mask32x4(self, a: &mut mask32x4<Self>, index: usize, value: bool) -> () {
5627 assert!(
5628 index < 4usize,
5629 "mask lane index {index} is out of bounds for {} lanes",
5630 4usize
5631 );
5632 let bit = 1u64 << index;
5633 let bits = u64::from((a).val);
5634 let bits = if value { bits | bit } else { bits & !bit };
5635 *a = mask32x4 {
5636 val: (bits) as _,
5637 simd: self,
5638 };
5639 }
5640 #[inline(always)]
5641 fn rotate_elements_left_mask32x4<const OFFSET: usize>(
5642 self,
5643 a: mask32x4<Self>,
5644 ) -> mask32x4<Self> {
5645 let bits = u64::from((a).val) & 15u64;
5646 let offset = OFFSET % 4;
5647 let bits = if offset == 0 {
5648 bits
5649 } else {
5650 ((bits >> offset) | (bits << (4 - offset))) & 15u64
5651 };
5652 mask32x4 {
5653 val: (bits) as _,
5654 simd: self,
5655 }
5656 }
5657 #[inline(always)]
5658 fn rotate_elements_right_mask32x4<const OFFSET: usize>(
5659 self,
5660 a: mask32x4<Self>,
5661 ) -> mask32x4<Self> {
5662 let bits = u64::from((a).val) & 15u64;
5663 let offset = OFFSET % 4;
5664 let bits = if offset == 0 {
5665 bits
5666 } else {
5667 ((bits << offset) | (bits >> (4 - offset))) & 15u64
5668 };
5669 mask32x4 {
5670 val: (bits) as _,
5671 simd: self,
5672 }
5673 }
5674 #[inline(always)]
5675 fn and_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5676 mask32x4 {
5677 val: ((u64::from((a).val) & u64::from((b).val)) & 15u64) as _,
5678 simd: self,
5679 }
5680 }
5681 #[inline(always)]
5682 fn or_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5683 mask32x4 {
5684 val: ((u64::from((a).val) | u64::from((b).val)) & 15u64) as _,
5685 simd: self,
5686 }
5687 }
5688 #[inline(always)]
5689 fn xor_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5690 mask32x4 {
5691 val: ((u64::from((a).val) ^ u64::from((b).val)) & 15u64) as _,
5692 simd: self,
5693 }
5694 }
5695 #[inline(always)]
5696 fn not_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
5697 mask32x4 {
5698 val: ((!u64::from((a).val)) & 15u64) as _,
5699 simd: self,
5700 }
5701 }
5702 #[inline(always)]
5703 fn reverse_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
5704 mask32x4 {
5705 val: (a.val.reverse_bits() >> 4usize) as _,
5706 simd: self,
5707 }
5708 }
5709 #[inline(always)]
5710 fn select_mask32x4(
5711 self,
5712 a: mask32x4<Self>,
5713 b: mask32x4<Self>,
5714 c: mask32x4<Self>,
5715 ) -> mask32x4<Self> {
5716 mask32x4 {
5717 val: (((u64::from((a).val) & u64::from((b).val))
5718 | ((!u64::from((a).val)) & u64::from((c).val)))
5719 & 15u64) as _,
5720 simd: self,
5721 }
5722 }
5723 #[inline(always)]
5724 fn simd_eq_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5725 mask32x4 {
5726 val: (!u64::from(a.val ^ b.val) & 15u64) as _,
5727 simd: self,
5728 }
5729 }
5730 #[inline(always)]
5731 fn simd_ne_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5732 mask32x4 {
5733 val: (u64::from(a.val ^ b.val) & 15u64) as _,
5734 simd: self,
5735 }
5736 }
5737 #[inline(always)]
5738 fn any_true_mask32x4(self, a: mask32x4<Self>) -> bool {
5739 let bits = u64::from((a).val) & 15u64;
5740 bits != 0
5741 }
5742 #[inline(always)]
5743 fn all_true_mask32x4(self, a: mask32x4<Self>) -> bool {
5744 let bits = u64::from((a).val) & 15u64;
5745 bits == 15u64
5746 }
5747 #[inline(always)]
5748 fn any_false_mask32x4(self, a: mask32x4<Self>) -> bool {
5749 let bits = u64::from((a).val) & 15u64;
5750 bits != 15u64
5751 }
5752 #[inline(always)]
5753 fn all_false_mask32x4(self, a: mask32x4<Self>) -> bool {
5754 let bits = u64::from((a).val) & 15u64;
5755 bits == 0
5756 }
5757 #[inline(always)]
5758 fn combine_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x8<Self> {
5759 let bits = (u64::from(a.val) | (u64::from(b.val) << 4usize)) & 255u64;
5760 mask32x8 {
5761 val: bits as _,
5762 simd: self,
5763 }
5764 }
5765 #[inline(always)]
5766 fn widen_mask32x4(self, a: mask32x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
5767 let bits = u64::from(a.val);
5768 (
5769 mask64x2 {
5770 val: (bits & 3u64) as _,
5771 simd: self,
5772 },
5773 mask64x2 {
5774 val: ((bits >> 2usize) & 3u64) as _,
5775 simd: self,
5776 },
5777 )
5778 }
5779 #[inline(always)]
5780 fn narrow_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask16x8<Self> {
5781 let low = u64::from(a.val) & 15u64;
5782 let high = u64::from(b.val) & 15u64;
5783 mask16x8 {
5784 val: (low | (high << 4usize)) as _,
5785 simd: self,
5786 }
5787 }
5788 #[inline(always)]
5789 fn abs_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5790 crate::kernel!(
5791 #[inline(always)]
5792 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
5793 _mm_andnot_pd(_mm_set1_pd(-0.0), a.into()).simd_into(token)
5794 }
5795 );
5796 kernel(self, a)
5797 }
5798 #[inline(always)]
5799 fn splat_f64x2(self, val: f64) -> f64x2<Self> {
5800 crate::kernel!(
5801 #[inline(always)]
5802 fn kernel(token: Avx512, val: f64) -> f64x2<Avx512> {
5803 _mm_set1_pd(val).simd_into(token)
5804 }
5805 );
5806 kernel(self, val)
5807 }
5808 #[inline(always)]
5809 fn reverse_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5810 let indices: u8x16<Self> =
5811 [8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7].simd_into(self);
5812 self.swizzle_dyn_f64x2(a, indices)
5813 }
5814 #[inline(always)]
5815 fn slide_f64x2<const SHIFT: usize>(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5816 if SHIFT >= 2usize {
5817 return b;
5818 }
5819 let result = dyn_alignr_128(
5820 self,
5821 Bytes::to_bytes(b).val.0,
5822 Bytes::to_bytes(a).val.0,
5823 SHIFT * 8usize,
5824 );
5825 Bytes::from_bytes(u8x16 {
5826 val: crate::support::Aligned128(result),
5827 simd: self,
5828 })
5829 }
5830 #[inline(always)]
5831 fn compress_f64x2(self, values: f64x2<Self>, mask: mask64x2<Self>) -> f64x2<Self> {
5832 crate::kernel!(
5833 #[inline(always)]
5834 fn kernel(
5835 token: Avx512,
5836 values: f64x2<Avx512>,
5837 mask: mask64x2<Avx512>,
5838 ) -> f64x2<Avx512> {
5839 _mm_maskz_compress_pd(u64::from((mask).val) as u8, values.into()).simd_into(token)
5840 }
5841 );
5842 kernel(self, values, mask)
5843 }
5844 #[inline(always)]
5845 fn compress_merge_f64x2(
5846 self,
5847 values: f64x2<Self>,
5848 mask: mask64x2<Self>,
5849 merge: f64x2<Self>,
5850 ) -> f64x2<Self> {
5851 crate::kernel!(
5852 #[inline(always)]
5853 fn kernel(
5854 token: Avx512,
5855 values: f64x2<Avx512>,
5856 mask: mask64x2<Avx512>,
5857 merge: f64x2<Avx512>,
5858 ) -> f64x2<Avx512> {
5859 _mm_mask_compress_pd(merge.into(), u64::from((mask).val) as u8, values.into())
5860 .simd_into(token)
5861 }
5862 );
5863 kernel(self, values, mask, merge)
5864 }
5865 #[inline(always)]
5866 fn expand_f64x2(self, values: f64x2<Self>, mask: mask64x2<Self>) -> f64x2<Self> {
5867 crate::kernel!(
5868 #[inline(always)]
5869 fn kernel(
5870 token: Avx512,
5871 values: f64x2<Avx512>,
5872 mask: mask64x2<Avx512>,
5873 ) -> f64x2<Avx512> {
5874 _mm_maskz_expand_pd(u64::from((mask).val) as u8, values.into()).simd_into(token)
5875 }
5876 );
5877 kernel(self, values, mask)
5878 }
5879 #[inline(always)]
5880 fn expand_merge_f64x2(
5881 self,
5882 values: f64x2<Self>,
5883 mask: mask64x2<Self>,
5884 merge: f64x2<Self>,
5885 ) -> f64x2<Self> {
5886 crate::kernel!(
5887 #[inline(always)]
5888 fn kernel(
5889 token: Avx512,
5890 values: f64x2<Avx512>,
5891 mask: mask64x2<Avx512>,
5892 merge: f64x2<Avx512>,
5893 ) -> f64x2<Avx512> {
5894 _mm_mask_expand_pd(merge.into(), u64::from((mask).val) as u8, values.into())
5895 .simd_into(token)
5896 }
5897 );
5898 kernel(self, values, mask, merge)
5899 }
5900 #[inline(always)]
5901 fn neg_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5902 crate::kernel!(
5903 #[inline(always)]
5904 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
5905 _mm_xor_pd(a.into(), _mm_set1_pd(-0.0)).simd_into(token)
5906 }
5907 );
5908 kernel(self, a)
5909 }
5910 #[inline(always)]
5911 fn sqrt_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5912 crate::kernel!(
5913 #[inline(always)]
5914 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
5915 _mm_sqrt_pd(a.into()).simd_into(token)
5916 }
5917 );
5918 kernel(self, a)
5919 }
5920 #[inline(always)]
5921 fn approximate_recip_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5922 crate::kernel!(
5923 #[inline(always)]
5924 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
5925 _mm_rcp14_pd(a.into()).simd_into(token)
5926 }
5927 );
5928 kernel(self, a)
5929 }
5930 #[inline(always)]
5931 fn add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5932 crate::kernel!(
5933 #[inline(always)]
5934 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
5935 _mm_add_pd(a.into(), b.into()).simd_into(token)
5936 }
5937 );
5938 kernel(self, a, b)
5939 }
5940 #[inline(always)]
5941 fn sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5942 crate::kernel!(
5943 #[inline(always)]
5944 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
5945 _mm_sub_pd(a.into(), b.into()).simd_into(token)
5946 }
5947 );
5948 kernel(self, a, b)
5949 }
5950 #[inline(always)]
5951 fn mul_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5952 crate::kernel!(
5953 #[inline(always)]
5954 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
5955 _mm_mul_pd(a.into(), b.into()).simd_into(token)
5956 }
5957 );
5958 kernel(self, a, b)
5959 }
5960 #[inline(always)]
5961 fn div_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5962 crate::kernel!(
5963 #[inline(always)]
5964 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
5965 _mm_div_pd(a.into(), b.into()).simd_into(token)
5966 }
5967 );
5968 kernel(self, a, b)
5969 }
5970 #[inline(always)]
5971 fn copysign_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5972 crate::kernel!(
5973 #[inline(always)]
5974 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
5975 let mask = _mm_set1_pd(-0.0);
5976 _mm_or_pd(_mm_and_pd(mask, b.into()), _mm_andnot_pd(mask, a.into()))
5977 .simd_into(token)
5978 }
5979 );
5980 kernel(self, a, b)
5981 }
5982 #[inline(always)]
5983 fn reduce_max_f64x2(self, a: f64x2<Self>) -> f64 {
5984 crate::kernel!(
5985 #[inline(always)]
5986 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64 {
5987 let reduced: __m128d = a.into();
5988 let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
5989 let reduced = { _mm_max_pd(reduced, shifted) };
5990 _mm_cvtsd_f64(reduced)
5991 }
5992 );
5993 kernel(self, a)
5994 }
5995 #[inline(always)]
5996 fn reduce_min_f64x2(self, a: f64x2<Self>) -> f64 {
5997 crate::kernel!(
5998 #[inline(always)]
5999 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64 {
6000 let reduced: __m128d = a.into();
6001 let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
6002 let reduced = { _mm_min_pd(reduced, shifted) };
6003 _mm_cvtsd_f64(reduced)
6004 }
6005 );
6006 kernel(self, a)
6007 }
6008 #[inline(always)]
6009 fn reduce_max_precise_f64x2(self, a: f64x2<Self>) -> f64 {
6010 crate::kernel!(
6011 #[inline(always)]
6012 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64 {
6013 let reduced: __m128d = a.into();
6014 let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
6015 let reduced = { _mm_range_pd::<5i32>(reduced, shifted) };
6016 _mm_cvtsd_f64(reduced)
6017 }
6018 );
6019 kernel(self, a)
6020 }
6021 #[inline(always)]
6022 fn reduce_min_precise_f64x2(self, a: f64x2<Self>) -> f64 {
6023 crate::kernel!(
6024 #[inline(always)]
6025 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64 {
6026 let reduced: __m128d = a.into();
6027 let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
6028 let reduced = { _mm_range_pd::<4i32>(reduced, shifted) };
6029 _mm_cvtsd_f64(reduced)
6030 }
6031 );
6032 kernel(self, a)
6033 }
6034 #[inline(always)]
6035 fn reduce_sum_f64x2(self, a: f64x2<Self>) -> f64 {
6036 crate::kernel!(
6037 #[inline(always)]
6038 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64 {
6039 let a: __m128d = a.into();
6040 _mm_cvtsd_f64(_mm_add_sd(a, _mm_unpackhi_pd(a, a)))
6041 }
6042 );
6043 kernel(self, a)
6044 }
6045 #[inline(always)]
6046 fn reduce_product_f64x2(self, a: f64x2<Self>) -> f64 {
6047 crate::kernel!(
6048 #[inline(always)]
6049 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64 {
6050 let a: __m128d = a.into();
6051 _mm_cvtsd_f64(_mm_mul_sd(a, _mm_unpackhi_pd(a, a)))
6052 }
6053 );
6054 kernel(self, a)
6055 }
6056 #[inline(always)]
6057 fn max_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6058 crate::kernel!(
6059 #[inline(always)]
6060 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
6061 _mm_max_pd(a.into(), b.into()).simd_into(token)
6062 }
6063 );
6064 kernel(self, a, b)
6065 }
6066 #[inline(always)]
6067 fn min_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6068 crate::kernel!(
6069 #[inline(always)]
6070 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
6071 _mm_min_pd(a.into(), b.into()).simd_into(token)
6072 }
6073 );
6074 kernel(self, a, b)
6075 }
6076 #[inline(always)]
6077 fn max_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6078 crate::kernel!(
6079 #[inline(always)]
6080 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
6081 _mm_range_pd::<5i32>(a.into(), b.into()).simd_into(token)
6082 }
6083 );
6084 kernel(self, a, b)
6085 }
6086 #[inline(always)]
6087 fn min_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6088 crate::kernel!(
6089 #[inline(always)]
6090 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
6091 _mm_range_pd::<4i32>(a.into(), b.into()).simd_into(token)
6092 }
6093 );
6094 kernel(self, a, b)
6095 }
6096 #[inline(always)]
6097 fn clamp_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6098 crate::kernel!(
6099 #[inline(always)]
6100 fn kernel(
6101 token: Avx512,
6102 a: f64x2<Avx512>,
6103 b: f64x2<Avx512>,
6104 c: f64x2<Avx512>,
6105 ) -> f64x2<Avx512> {
6106 debug_assert!(
6107 b.simd_le(c).all_true(),
6108 "clamp requires min <= max for all elements"
6109 );
6110 _mm_min_pd(c.into(), _mm_max_pd(b.into(), a.into())).simd_into(token)
6111 }
6112 );
6113 kernel(self, a, b, c)
6114 }
6115 #[inline(always)]
6116 fn simd_eq_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
6117 crate::kernel!(
6118 #[inline(always)]
6119 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> mask64x2<Avx512> {
6120 mask64x2 {
6121 val: _mm_cmp_pd_mask::<0i32>(a.into(), b.into()),
6122 simd: token,
6123 }
6124 }
6125 );
6126 kernel(self, a, b)
6127 }
6128 #[inline(always)]
6129 fn simd_ne_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
6130 crate::kernel!(
6131 #[inline(always)]
6132 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> mask64x2<Avx512> {
6133 mask64x2 {
6134 val: _mm_cmp_pd_mask::<4i32>(a.into(), b.into()),
6135 simd: token,
6136 }
6137 }
6138 );
6139 kernel(self, a, b)
6140 }
6141 #[inline(always)]
6142 fn simd_lt_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
6143 crate::kernel!(
6144 #[inline(always)]
6145 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> mask64x2<Avx512> {
6146 mask64x2 {
6147 val: _mm_cmp_pd_mask::<17i32>(a.into(), b.into()),
6148 simd: token,
6149 }
6150 }
6151 );
6152 kernel(self, a, b)
6153 }
6154 #[inline(always)]
6155 fn simd_le_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
6156 crate::kernel!(
6157 #[inline(always)]
6158 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> mask64x2<Avx512> {
6159 mask64x2 {
6160 val: _mm_cmp_pd_mask::<18i32>(a.into(), b.into()),
6161 simd: token,
6162 }
6163 }
6164 );
6165 kernel(self, a, b)
6166 }
6167 #[inline(always)]
6168 fn zip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6169 crate::kernel!(
6170 #[inline(always)]
6171 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
6172 _mm_unpacklo_pd(a.into(), b.into()).simd_into(token)
6173 }
6174 );
6175 kernel(self, a, b)
6176 }
6177 #[inline(always)]
6178 fn zip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6179 crate::kernel!(
6180 #[inline(always)]
6181 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
6182 _mm_unpackhi_pd(a.into(), b.into()).simd_into(token)
6183 }
6184 );
6185 kernel(self, a, b)
6186 }
6187 #[inline(always)]
6188 fn unzip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6189 crate::kernel!(
6190 #[inline(always)]
6191 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
6192 _mm_shuffle_pd::<0b00>(a.into(), b.into()).simd_into(token)
6193 }
6194 );
6195 kernel(self, a, b)
6196 }
6197 #[inline(always)]
6198 fn unzip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6199 crate::kernel!(
6200 #[inline(always)]
6201 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
6202 _mm_shuffle_pd::<0b11>(a.into(), b.into()).simd_into(token)
6203 }
6204 );
6205 kernel(self, a, b)
6206 }
6207 #[inline(always)]
6208 fn interleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
6209 (self.zip_low_f64x2(a, b), self.zip_high_f64x2(a, b))
6210 }
6211 #[inline(always)]
6212 fn deinterleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
6213 (self.unzip_low_f64x2(a, b), self.unzip_high_f64x2(a, b))
6214 }
6215 #[inline(always)]
6216 fn mul_add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6217 crate::kernel!(
6218 #[inline(always)]
6219 fn kernel(
6220 token: Avx512,
6221 a: f64x2<Avx512>,
6222 b: f64x2<Avx512>,
6223 c: f64x2<Avx512>,
6224 ) -> f64x2<Avx512> {
6225 _mm_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token)
6226 }
6227 );
6228 kernel(self, a, b, c)
6229 }
6230 #[inline(always)]
6231 fn mul_add_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6232 self.mul_add_f64x2(a, b, c)
6233 }
6234 #[inline(always)]
6235 fn mul_sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6236 crate::kernel!(
6237 #[inline(always)]
6238 fn kernel(
6239 token: Avx512,
6240 a: f64x2<Avx512>,
6241 b: f64x2<Avx512>,
6242 c: f64x2<Avx512>,
6243 ) -> f64x2<Avx512> {
6244 _mm_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token)
6245 }
6246 );
6247 kernel(self, a, b, c)
6248 }
6249 #[inline(always)]
6250 fn mul_sub_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6251 self.mul_sub_f64x2(a, b, c)
6252 }
6253 #[inline(always)]
6254 fn floor_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6255 crate::kernel!(
6256 #[inline(always)]
6257 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
6258 _mm_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
6259 .simd_into(token)
6260 }
6261 );
6262 kernel(self, a)
6263 }
6264 #[inline(always)]
6265 fn ceil_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6266 crate::kernel!(
6267 #[inline(always)]
6268 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
6269 _mm_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
6270 .simd_into(token)
6271 }
6272 );
6273 kernel(self, a)
6274 }
6275 #[inline(always)]
6276 fn round_ties_even_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6277 crate::kernel!(
6278 #[inline(always)]
6279 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
6280 _mm_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
6281 .simd_into(token)
6282 }
6283 );
6284 kernel(self, a)
6285 }
6286 #[inline(always)]
6287 fn round_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6288 let bias = self.splat_f64x2(const { f64::next_down(0.5) });
6289 (a + self.copysign_f64x2(bias, a)).trunc()
6290 }
6291 #[inline(always)]
6292 fn fract_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6293 a - self.trunc_f64x2(a)
6294 }
6295 #[inline(always)]
6296 fn trunc_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6297 crate::kernel!(
6298 #[inline(always)]
6299 fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
6300 _mm_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
6301 .simd_into(token)
6302 }
6303 );
6304 kernel(self, a)
6305 }
6306 #[inline(always)]
6307 fn to_degrees_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6308 a * const { f64::to_degrees(1.0) }
6309 }
6310 #[inline(always)]
6311 fn to_radians_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6312 a * const { f64::to_radians(1.0) }
6313 }
6314 #[inline(always)]
6315 fn is_nan_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6316 crate::kernel!(
6317 #[inline(always)]
6318 fn kernel(token: Avx512, a: f64x2<Avx512>) -> mask64x2<Avx512> {
6319 mask64x2 {
6320 val: _mm_fpclass_pd_mask::<129i32>(a.into()),
6321 simd: token,
6322 }
6323 }
6324 );
6325 kernel(self, a)
6326 }
6327 #[inline(always)]
6328 fn is_infinite_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6329 crate::kernel!(
6330 #[inline(always)]
6331 fn kernel(token: Avx512, a: f64x2<Avx512>) -> mask64x2<Avx512> {
6332 mask64x2 {
6333 val: _mm_fpclass_pd_mask::<24i32>(a.into()),
6334 simd: token,
6335 }
6336 }
6337 );
6338 kernel(self, a)
6339 }
6340 #[inline(always)]
6341 fn is_finite_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6342 crate::kernel!(
6343 #[inline(always)]
6344 fn kernel(token: Avx512, a: f64x2<Avx512>) -> mask64x2<Avx512> {
6345 !(mask64x2 {
6346 val: _mm_fpclass_pd_mask::<153i32>(a.into()),
6347 simd: token,
6348 })
6349 }
6350 );
6351 kernel(self, a)
6352 }
6353 #[inline(always)]
6354 fn is_subnormal_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6355 crate::kernel!(
6356 #[inline(always)]
6357 fn kernel(token: Avx512, a: f64x2<Avx512>) -> mask64x2<Avx512> {
6358 mask64x2 {
6359 val: _mm_fpclass_pd_mask::<129i32>(_mm_xor_pd(
6360 a.into(),
6361 _mm_set1_pd(f64::INFINITY),
6362 )),
6363 simd: token,
6364 }
6365 }
6366 );
6367 kernel(self, a)
6368 }
6369 #[inline(always)]
6370 fn is_normal_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6371 crate::kernel!(
6372 #[inline(always)]
6373 fn kernel(token: Avx512, a: f64x2<Avx512>) -> mask64x2<Avx512> {
6374 !(mask64x2 {
6375 val: _mm_fpclass_pd_mask::<191i32>(a.into()),
6376 simd: token,
6377 })
6378 }
6379 );
6380 kernel(self, a)
6381 }
6382 #[inline(always)]
6383 fn is_sign_positive_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6384 let i: i64x2<Self> = a.bitcast();
6385 (i & i64::MIN).simd_eq(0)
6386 }
6387 #[inline(always)]
6388 fn is_sign_negative_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6389 !a.is_sign_positive()
6390 }
6391 #[inline(always)]
6392 fn select_f64x2(self, a: mask64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6393 crate::kernel!(
6394 #[inline(always)]
6395 fn kernel(
6396 token: Avx512,
6397 a: mask64x2<Avx512>,
6398 b: f64x2<Avx512>,
6399 c: f64x2<Avx512>,
6400 ) -> f64x2<Avx512> {
6401 _mm_mask_blend_pd(a.val, c.into(), b.into()).simd_into(token)
6402 }
6403 );
6404 kernel(self, a, b, c)
6405 }
6406 #[inline(always)]
6407 fn combine_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x4<Self> {
6408 crate::kernel!(
6409 #[inline(always)]
6410 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x4<Avx512> {
6411 _mm256_setr_m128d(a.into(), b.into()).simd_into(token)
6412 }
6413 );
6414 kernel(self, a, b)
6415 }
6416 #[inline(always)]
6417 fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2<Self>; 4usize] {
6418 crate::kernel!(
6419 #[inline(always)]
6420 fn kernel(token: Avx512, src: &[f64; 8usize]) -> [f64x2<Avx512>; 4usize] {
6421 let lanes: __m512d =
6422 crate::transmute::checked_transmute_copy::<[f64; 8usize], __m512d>(src);
6423 let lanes = _mm512_permutexvar_pd(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes);
6424 let outputs: [__m128d; 4] = crate::transmute::checked_transmute_copy(&lanes);
6425 [
6426 outputs[0].simd_into(token),
6427 outputs[1].simd_into(token),
6428 outputs[2].simd_into(token),
6429 outputs[3].simd_into(token),
6430 ]
6431 }
6432 );
6433 kernel(self, src)
6434 }
6435 #[inline(always)]
6436 fn store_four_interleaved_f64x2(
6437 self,
6438 vectors: [f64x2<Self>; 4usize],
6439 dest: &mut [f64; 8usize],
6440 ) -> () {
6441 crate::kernel!(
6442 #[inline(always)]
6443 fn kernel(
6444 token: Avx512,
6445 vectors: [f64x2<Avx512>; 4usize],
6446 dest: &mut [f64; 8usize],
6447 ) -> () {
6448 let _ = token;
6449 let inputs: [__m128d; 4] = [
6450 vectors[0].into(),
6451 vectors[1].into(),
6452 vectors[2].into(),
6453 vectors[3].into(),
6454 ];
6455 let lanes: __m512d = crate::transmute::checked_transmute_copy(&inputs);
6456 let lanes = _mm512_permutexvar_pd(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), lanes);
6457 crate::transmute::checked_transmute_store::<__m512d, [f64; 8usize]>(lanes, dest);
6458 }
6459 );
6460 kernel(self, vectors, dest);
6461 }
6462 #[inline(always)]
6463 fn narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6464 crate::kernel!(
6465 #[inline(always)]
6466 fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f32x4<Avx512> {
6467 let low = _mm_cvtpd_ps(a.into());
6468 let high = _mm_cvtpd_ps(b.into());
6469 _mm_movelh_ps(low, high).simd_into(token)
6470 }
6471 );
6472 kernel(self, a, b)
6473 }
6474 #[inline(always)]
6475 fn saturating_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6476 self.narrow_f64x2(a, b)
6477 }
6478 #[inline(always)]
6479 fn relaxed_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6480 self.narrow_f64x2(a, b)
6481 }
6482 #[inline(always)]
6483 fn cvt_u64_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
6484 crate::kernel!(
6485 #[inline(always)]
6486 fn kernel(token: Avx512, a: f64x2<Avx512>) -> u64x2<Avx512> {
6487 _mm_cvttpd_epu64(a.into()).simd_into(token)
6488 }
6489 );
6490 kernel(self, a)
6491 }
6492 #[inline(always)]
6493 fn cvt_u64_precise_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
6494 crate::kernel!(
6495 #[inline(always)]
6496 fn kernel(token: Avx512, a: f64x2<Avx512>) -> u64x2<Avx512> {
6497 let a = a.into();
6498 let positive = _mm_cmp_pd_mask::<17i32>(_mm_setzero_pd(), a);
6499 _mm_maskz_cvttpd_epu64(positive, a).simd_into(token)
6500 }
6501 );
6502 kernel(self, a)
6503 }
6504 #[inline(always)]
6505 fn cvt_i64_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
6506 crate::kernel!(
6507 #[inline(always)]
6508 fn kernel(token: Avx512, a: f64x2<Avx512>) -> i64x2<Avx512> {
6509 _mm_cvttpd_epi64(a.into()).simd_into(token)
6510 }
6511 );
6512 kernel(self, a)
6513 }
6514 #[inline(always)]
6515 fn cvt_i64_precise_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
6516 crate::kernel!(
6517 #[inline(always)]
6518 fn kernel(token: Avx512, a: f64x2<Avx512>) -> i64x2<Avx512> {
6519 let a = a.into();
6520 let in_range = _mm_cmp_pd_mask::<17i32>(a, _mm_set1_pd(9223372036854775808.0));
6521 let mut converted = _mm_mask_cvttpd_epi64(_mm_set1_epi64x(i64::MAX), in_range, a);
6522 let is_not_nan = _mm_cmp_pd_mask::<7i32>(a, a);
6523 converted = _mm_mask_blend_epi64(is_not_nan, _mm_setzero_si128(), converted);
6524 converted.simd_into(token)
6525 }
6526 );
6527 kernel(self, a)
6528 }
6529 #[inline(always)]
6530 fn abs_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6531 crate::kernel!(
6532 #[inline(always)]
6533 fn kernel(token: Avx512, a: i64x2<Avx512>) -> i64x2<Avx512> {
6534 _mm_abs_epi64(a.into()).simd_into(token)
6535 }
6536 );
6537 kernel(self, a)
6538 }
6539 #[inline(always)]
6540 fn splat_i64x2(self, val: i64) -> i64x2<Self> {
6541 crate::kernel!(
6542 #[inline(always)]
6543 fn kernel(token: Avx512, val: i64) -> i64x2<Avx512> {
6544 _mm_set1_epi64x(val).simd_into(token)
6545 }
6546 );
6547 kernel(self, val)
6548 }
6549 #[inline(always)]
6550 fn reverse_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6551 let indices: u8x16<Self> =
6552 [8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7].simd_into(self);
6553 self.swizzle_dyn_i64x2(a, indices)
6554 }
6555 #[inline(always)]
6556 fn slide_i64x2<const SHIFT: usize>(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6557 if SHIFT >= 2usize {
6558 return b;
6559 }
6560 let result = dyn_alignr_128(
6561 self,
6562 Bytes::to_bytes(b).val.0,
6563 Bytes::to_bytes(a).val.0,
6564 SHIFT * 8usize,
6565 );
6566 Bytes::from_bytes(u8x16 {
6567 val: crate::support::Aligned128(result),
6568 simd: self,
6569 })
6570 }
6571 #[inline(always)]
6572 fn compress_i64x2(self, values: i64x2<Self>, mask: mask64x2<Self>) -> i64x2<Self> {
6573 crate::kernel!(
6574 #[inline(always)]
6575 fn kernel(
6576 token: Avx512,
6577 values: i64x2<Avx512>,
6578 mask: mask64x2<Avx512>,
6579 ) -> i64x2<Avx512> {
6580 _mm_maskz_compress_epi64(u64::from((mask).val) as u8, values.into())
6581 .simd_into(token)
6582 }
6583 );
6584 kernel(self, values, mask)
6585 }
6586 #[inline(always)]
6587 fn compress_merge_i64x2(
6588 self,
6589 values: i64x2<Self>,
6590 mask: mask64x2<Self>,
6591 merge: i64x2<Self>,
6592 ) -> i64x2<Self> {
6593 crate::kernel!(
6594 #[inline(always)]
6595 fn kernel(
6596 token: Avx512,
6597 values: i64x2<Avx512>,
6598 mask: mask64x2<Avx512>,
6599 merge: i64x2<Avx512>,
6600 ) -> i64x2<Avx512> {
6601 _mm_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into())
6602 .simd_into(token)
6603 }
6604 );
6605 kernel(self, values, mask, merge)
6606 }
6607 #[inline(always)]
6608 fn expand_i64x2(self, values: i64x2<Self>, mask: mask64x2<Self>) -> i64x2<Self> {
6609 crate::kernel!(
6610 #[inline(always)]
6611 fn kernel(
6612 token: Avx512,
6613 values: i64x2<Avx512>,
6614 mask: mask64x2<Avx512>,
6615 ) -> i64x2<Avx512> {
6616 _mm_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()).simd_into(token)
6617 }
6618 );
6619 kernel(self, values, mask)
6620 }
6621 #[inline(always)]
6622 fn expand_merge_i64x2(
6623 self,
6624 values: i64x2<Self>,
6625 mask: mask64x2<Self>,
6626 merge: i64x2<Self>,
6627 ) -> i64x2<Self> {
6628 crate::kernel!(
6629 #[inline(always)]
6630 fn kernel(
6631 token: Avx512,
6632 values: i64x2<Avx512>,
6633 mask: mask64x2<Avx512>,
6634 merge: i64x2<Avx512>,
6635 ) -> i64x2<Avx512> {
6636 _mm_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into())
6637 .simd_into(token)
6638 }
6639 );
6640 kernel(self, values, mask, merge)
6641 }
6642 #[inline(always)]
6643 fn count_ones_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6644 crate::kernel!(
6645 #[inline(always)]
6646 fn kernel(token: Avx512, a: i64x2<Avx512>) -> i64x2<Avx512> {
6647 _mm_popcnt_epi64(a.into()).simd_into(token)
6648 }
6649 );
6650 kernel(self, a)
6651 }
6652 #[inline(always)]
6653 fn count_zeros_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6654 self.count_ones_i64x2(self.not_i64x2(a))
6655 }
6656 #[inline(always)]
6657 fn add_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6658 crate::kernel!(
6659 #[inline(always)]
6660 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6661 _mm_add_epi64(a.into(), b.into()).simd_into(token)
6662 }
6663 );
6664 kernel(self, a, b)
6665 }
6666 #[inline(always)]
6667 fn saturating_add_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6668 crate::kernel!(
6669 #[inline(always)]
6670 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6671 let a = a.into();
6672 let b = b.into();
6673 let wrapped = _mm_add_epi64(a, b);
6674 let overflow_bits = _mm_ternarylogic_epi64::<0x42>(a, b, wrapped);
6675 let overflow_mask = _mm_srai_epi64::<63>(overflow_bits);
6676 let direction = _mm_add_epi64(_mm_srli_epi64::<63>(a), _mm_set1_epi64x(i64::MAX));
6677 _mm_ternarylogic_epi64::<0xca>(overflow_mask, direction, wrapped).simd_into(token)
6678 }
6679 );
6680 kernel(self, a, b)
6681 }
6682 #[inline(always)]
6683 fn sub_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6684 crate::kernel!(
6685 #[inline(always)]
6686 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6687 _mm_sub_epi64(a.into(), b.into()).simd_into(token)
6688 }
6689 );
6690 kernel(self, a, b)
6691 }
6692 #[inline(always)]
6693 fn saturating_sub_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6694 crate::kernel!(
6695 #[inline(always)]
6696 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6697 let a = a.into();
6698 let b = b.into();
6699 let wrapped = _mm_sub_epi64(a, b);
6700 let overflow_bits = _mm_ternarylogic_epi64::<0x18>(a, b, wrapped);
6701 let overflow_mask = _mm_srai_epi64::<63>(overflow_bits);
6702 let direction = _mm_add_epi64(_mm_srli_epi64::<63>(a), _mm_set1_epi64x(i64::MAX));
6703 _mm_ternarylogic_epi64::<0xca>(overflow_mask, direction, wrapped).simd_into(token)
6704 }
6705 );
6706 kernel(self, a, b)
6707 }
6708 #[inline(always)]
6709 fn mul_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6710 crate::kernel!(
6711 #[inline(always)]
6712 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6713 _mm_mullo_epi64(a.into(), b.into()).simd_into(token)
6714 }
6715 );
6716 kernel(self, a, b)
6717 }
6718 #[inline(always)]
6719 fn and_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6720 crate::kernel!(
6721 #[inline(always)]
6722 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6723 _mm_and_si128(a.into(), b.into()).simd_into(token)
6724 }
6725 );
6726 kernel(self, a, b)
6727 }
6728 #[inline(always)]
6729 fn or_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6730 crate::kernel!(
6731 #[inline(always)]
6732 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6733 _mm_or_si128(a.into(), b.into()).simd_into(token)
6734 }
6735 );
6736 kernel(self, a, b)
6737 }
6738 #[inline(always)]
6739 fn xor_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6740 crate::kernel!(
6741 #[inline(always)]
6742 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6743 _mm_xor_si128(a.into(), b.into()).simd_into(token)
6744 }
6745 );
6746 kernel(self, a, b)
6747 }
6748 #[inline(always)]
6749 fn not_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6750 a ^ !0
6751 }
6752 #[inline(always)]
6753 fn shl_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
6754 crate::kernel!(
6755 #[inline(always)]
6756 fn kernel(token: Avx512, a: i64x2<Avx512>, shift: u32) -> i64x2<Avx512> {
6757 _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
6758 }
6759 );
6760 kernel(self, a, shift)
6761 }
6762 #[inline(always)]
6763 fn shlv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6764 crate::kernel!(
6765 #[inline(always)]
6766 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6767 _mm_sllv_epi64(a.into(), b.into()).simd_into(token)
6768 }
6769 );
6770 kernel(self, a, b)
6771 }
6772 #[inline(always)]
6773 fn shr_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
6774 crate::kernel!(
6775 #[inline(always)]
6776 fn kernel(token: Avx512, a: i64x2<Avx512>, shift: u32) -> i64x2<Avx512> {
6777 _mm_sra_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
6778 }
6779 );
6780 kernel(self, a, shift)
6781 }
6782 #[inline(always)]
6783 fn shrv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6784 crate::kernel!(
6785 #[inline(always)]
6786 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6787 _mm_srav_epi64(a.into(), b.into()).simd_into(token)
6788 }
6789 );
6790 kernel(self, a, b)
6791 }
6792 #[inline(always)]
6793 fn reduce_max_i64x2(self, a: i64x2<Self>) -> i64 {
6794 crate::kernel!(
6795 #[inline(always)]
6796 fn kernel(token: Avx512, a: i64x2<Avx512>) -> i64 {
6797 let lanes: [i64; 2] = a.into();
6798 lanes[0].max(lanes[1])
6799 }
6800 );
6801 kernel(self, a)
6802 }
6803 #[inline(always)]
6804 fn reduce_min_i64x2(self, a: i64x2<Self>) -> i64 {
6805 crate::kernel!(
6806 #[inline(always)]
6807 fn kernel(token: Avx512, a: i64x2<Avx512>) -> i64 {
6808 let lanes: [i64; 2] = a.into();
6809 lanes[0].min(lanes[1])
6810 }
6811 );
6812 kernel(self, a)
6813 }
6814 #[inline(always)]
6815 fn reduce_sum_i64x2(self, a: i64x2<Self>) -> i64 {
6816 crate::kernel!(
6817 #[inline(always)]
6818 fn kernel(token: Avx512, a: i64x2<Avx512>) -> i64 {
6819 let sum: __m128i = a.into();
6820 let sum = _mm_add_epi64(sum, _mm_srli_si128::<8>(sum));
6821 let lanes: [i64; 2usize] = crate::transmute::checked_transmute_copy(&sum);
6822 lanes[0]
6823 }
6824 );
6825 kernel(self, a)
6826 }
6827 #[inline(always)]
6828 fn reduce_product_i64x2(self, a: i64x2<Self>) -> i64 {
6829 let mul_level_0: [i64; 1usize] = [a[0usize].wrapping_mul(a[1usize])];
6830 mul_level_0[0]
6831 }
6832 #[inline(always)]
6833 fn max_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6834 crate::kernel!(
6835 #[inline(always)]
6836 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6837 _mm_max_epi64(a.into(), b.into()).simd_into(token)
6838 }
6839 );
6840 kernel(self, a, b)
6841 }
6842 #[inline(always)]
6843 fn min_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6844 crate::kernel!(
6845 #[inline(always)]
6846 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6847 _mm_min_epi64(a.into(), b.into()).simd_into(token)
6848 }
6849 );
6850 kernel(self, a, b)
6851 }
6852 #[inline(always)]
6853 fn clamp_i64x2(self, a: i64x2<Self>, b: i64x2<Self>, c: i64x2<Self>) -> i64x2<Self> {
6854 debug_assert!(
6855 b.simd_le(c).all_true(),
6856 "clamp requires min <= max for all elements"
6857 );
6858 a.max(b).min(c)
6859 }
6860 #[inline(always)]
6861 fn simd_eq_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6862 crate::kernel!(
6863 #[inline(always)]
6864 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> mask64x2<Avx512> {
6865 mask64x2 {
6866 val: _mm_cmpeq_epi64_mask(a.into(), b.into()),
6867 simd: token,
6868 }
6869 }
6870 );
6871 kernel(self, a, b)
6872 }
6873 #[inline(always)]
6874 fn simd_ne_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6875 crate::kernel!(
6876 #[inline(always)]
6877 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> mask64x2<Avx512> {
6878 mask64x2 {
6879 val: _mm_cmpneq_epi64_mask(a.into(), b.into()),
6880 simd: token,
6881 }
6882 }
6883 );
6884 kernel(self, a, b)
6885 }
6886 #[inline(always)]
6887 fn simd_lt_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6888 crate::kernel!(
6889 #[inline(always)]
6890 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> mask64x2<Avx512> {
6891 mask64x2 {
6892 val: _mm_cmplt_epi64_mask(a.into(), b.into()),
6893 simd: token,
6894 }
6895 }
6896 );
6897 kernel(self, a, b)
6898 }
6899 #[inline(always)]
6900 fn simd_le_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6901 crate::kernel!(
6902 #[inline(always)]
6903 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> mask64x2<Avx512> {
6904 mask64x2 {
6905 val: _mm_cmple_epi64_mask(a.into(), b.into()),
6906 simd: token,
6907 }
6908 }
6909 );
6910 kernel(self, a, b)
6911 }
6912 #[inline(always)]
6913 fn zip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6914 crate::kernel!(
6915 #[inline(always)]
6916 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6917 _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
6918 }
6919 );
6920 kernel(self, a, b)
6921 }
6922 #[inline(always)]
6923 fn zip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6924 crate::kernel!(
6925 #[inline(always)]
6926 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6927 _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
6928 }
6929 );
6930 kernel(self, a, b)
6931 }
6932 #[inline(always)]
6933 fn unzip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6934 crate::kernel!(
6935 #[inline(always)]
6936 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6937 _mm_permutex2var_epi64(a.into(), _mm_set_epi64x(2, 0), b.into()).simd_into(token)
6938 }
6939 );
6940 kernel(self, a, b)
6941 }
6942 #[inline(always)]
6943 fn unzip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6944 crate::kernel!(
6945 #[inline(always)]
6946 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x2<Avx512> {
6947 _mm_permutex2var_epi64(a.into(), _mm_set_epi64x(3, 1), b.into()).simd_into(token)
6948 }
6949 );
6950 kernel(self, a, b)
6951 }
6952 #[inline(always)]
6953 fn interleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
6954 (self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b))
6955 }
6956 #[inline(always)]
6957 fn deinterleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
6958 crate::kernel!(
6959 #[inline(always)]
6960 fn kernel(
6961 token: Avx512,
6962 a: i64x2<Avx512>,
6963 b: i64x2<Avx512>,
6964 ) -> (i64x2<Avx512>, i64x2<Avx512>) {
6965 let a = a.into();
6966 let b = b.into();
6967 (
6968 _mm_permutex2var_epi64(a, _mm_set_epi64x(2, 0), b).simd_into(token),
6969 _mm_permutex2var_epi64(a, _mm_set_epi64x(3, 1), b).simd_into(token),
6970 )
6971 }
6972 );
6973 kernel(self, a, b)
6974 }
6975 #[inline(always)]
6976 fn select_i64x2(self, a: mask64x2<Self>, b: i64x2<Self>, c: i64x2<Self>) -> i64x2<Self> {
6977 crate::kernel!(
6978 #[inline(always)]
6979 fn kernel(
6980 token: Avx512,
6981 a: mask64x2<Avx512>,
6982 b: i64x2<Avx512>,
6983 c: i64x2<Avx512>,
6984 ) -> i64x2<Avx512> {
6985 _mm_mask_blend_epi64(a.val, c.into(), b.into()).simd_into(token)
6986 }
6987 );
6988 kernel(self, a, b, c)
6989 }
6990 #[inline(always)]
6991 fn combine_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x4<Self> {
6992 crate::kernel!(
6993 #[inline(always)]
6994 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i64x4<Avx512> {
6995 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
6996 }
6997 );
6998 kernel(self, a, b)
6999 }
7000 #[inline(always)]
7001 fn neg_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
7002 crate::kernel!(
7003 #[inline(always)]
7004 fn kernel(token: Avx512, a: i64x2<Avx512>) -> i64x2<Avx512> {
7005 _mm_sub_epi64(_mm_setzero_si128(), a.into()).simd_into(token)
7006 }
7007 );
7008 kernel(self, a)
7009 }
7010 #[inline(always)]
7011 fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2<Self>; 4usize] {
7012 crate::kernel!(
7013 #[inline(always)]
7014 fn kernel(token: Avx512, src: &[i64; 8usize]) -> [i64x2<Avx512>; 4usize] {
7015 let lanes: __m512i =
7016 crate::transmute::checked_transmute_copy::<[i64; 8usize], __m512i>(src);
7017 let lanes =
7018 _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes);
7019 let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes);
7020 [
7021 outputs[0].simd_into(token),
7022 outputs[1].simd_into(token),
7023 outputs[2].simd_into(token),
7024 outputs[3].simd_into(token),
7025 ]
7026 }
7027 );
7028 kernel(self, src)
7029 }
7030 #[inline(always)]
7031 fn store_four_interleaved_i64x2(
7032 self,
7033 vectors: [i64x2<Self>; 4usize],
7034 dest: &mut [i64; 8usize],
7035 ) -> () {
7036 crate::kernel!(
7037 #[inline(always)]
7038 fn kernel(
7039 token: Avx512,
7040 vectors: [i64x2<Avx512>; 4usize],
7041 dest: &mut [i64; 8usize],
7042 ) -> () {
7043 let _ = token;
7044 let inputs: [__m128i; 4] = [
7045 vectors[0].into(),
7046 vectors[1].into(),
7047 vectors[2].into(),
7048 vectors[3].into(),
7049 ];
7050 let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs);
7051 let lanes =
7052 _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), lanes);
7053 crate::transmute::checked_transmute_store::<__m512i, [i64; 8usize]>(lanes, dest);
7054 }
7055 );
7056 kernel(self, vectors, dest);
7057 }
7058 #[inline(always)]
7059 fn narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
7060 crate::kernel!(
7061 #[inline(always)]
7062 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i32x4<Avx512> {
7063 let low = _mm_cvtepi64_epi32(a.into());
7064 let high = _mm_cvtepi64_epi32(b.into());
7065 _mm_unpacklo_epi64(low, high).simd_into(token)
7066 }
7067 );
7068 kernel(self, a, b)
7069 }
7070 #[inline(always)]
7071 fn saturating_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
7072 crate::kernel!(
7073 #[inline(always)]
7074 fn kernel(token: Avx512, a: i64x2<Avx512>, b: i64x2<Avx512>) -> i32x4<Avx512> {
7075 let low = _mm_cvtsepi64_epi32(a.into());
7076 let high = _mm_cvtsepi64_epi32(b.into());
7077 _mm_unpacklo_epi64(low, high).simd_into(token)
7078 }
7079 );
7080 kernel(self, a, b)
7081 }
7082 #[inline(always)]
7083 fn relaxed_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
7084 debug_assert!(
7085 a.as_slice()
7086 .iter()
7087 .chain(b.as_slice())
7088 .all(|&value| { value >= i32::MIN as i64 && value <= i32::MAX as i64 }),
7089 "relaxed_narrow inputs must fit in the destination type",
7090 );
7091 self.narrow_i64x2(a, b)
7092 }
7093 #[inline(always)]
7094 fn cvt_f64_i64x2(self, a: i64x2<Self>) -> f64x2<Self> {
7095 crate::kernel!(
7096 #[inline(always)]
7097 fn kernel(token: Avx512, a: i64x2<Avx512>) -> f64x2<Avx512> {
7098 _mm_cvtepi64_pd(a.into()).simd_into(token)
7099 }
7100 );
7101 kernel(self, a)
7102 }
7103 #[inline(always)]
7104 fn splat_u64x2(self, val: u64) -> u64x2<Self> {
7105 crate::kernel!(
7106 #[inline(always)]
7107 fn kernel(token: Avx512, val: u64) -> u64x2<Avx512> {
7108 _mm_set1_epi64x(val.cast_signed()).simd_into(token)
7109 }
7110 );
7111 kernel(self, val)
7112 }
7113 #[inline(always)]
7114 fn reverse_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
7115 let indices: u8x16<Self> =
7116 [8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7].simd_into(self);
7117 self.swizzle_dyn_u64x2(a, indices)
7118 }
7119 #[inline(always)]
7120 fn slide_u64x2<const SHIFT: usize>(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7121 if SHIFT >= 2usize {
7122 return b;
7123 }
7124 let result = dyn_alignr_128(
7125 self,
7126 Bytes::to_bytes(b).val.0,
7127 Bytes::to_bytes(a).val.0,
7128 SHIFT * 8usize,
7129 );
7130 Bytes::from_bytes(u8x16 {
7131 val: crate::support::Aligned128(result),
7132 simd: self,
7133 })
7134 }
7135 #[inline(always)]
7136 fn compress_u64x2(self, values: u64x2<Self>, mask: mask64x2<Self>) -> u64x2<Self> {
7137 crate::kernel!(
7138 #[inline(always)]
7139 fn kernel(
7140 token: Avx512,
7141 values: u64x2<Avx512>,
7142 mask: mask64x2<Avx512>,
7143 ) -> u64x2<Avx512> {
7144 _mm_maskz_compress_epi64(u64::from((mask).val) as u8, values.into())
7145 .simd_into(token)
7146 }
7147 );
7148 kernel(self, values, mask)
7149 }
7150 #[inline(always)]
7151 fn compress_merge_u64x2(
7152 self,
7153 values: u64x2<Self>,
7154 mask: mask64x2<Self>,
7155 merge: u64x2<Self>,
7156 ) -> u64x2<Self> {
7157 crate::kernel!(
7158 #[inline(always)]
7159 fn kernel(
7160 token: Avx512,
7161 values: u64x2<Avx512>,
7162 mask: mask64x2<Avx512>,
7163 merge: u64x2<Avx512>,
7164 ) -> u64x2<Avx512> {
7165 _mm_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into())
7166 .simd_into(token)
7167 }
7168 );
7169 kernel(self, values, mask, merge)
7170 }
7171 #[inline(always)]
7172 fn expand_u64x2(self, values: u64x2<Self>, mask: mask64x2<Self>) -> u64x2<Self> {
7173 crate::kernel!(
7174 #[inline(always)]
7175 fn kernel(
7176 token: Avx512,
7177 values: u64x2<Avx512>,
7178 mask: mask64x2<Avx512>,
7179 ) -> u64x2<Avx512> {
7180 _mm_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()).simd_into(token)
7181 }
7182 );
7183 kernel(self, values, mask)
7184 }
7185 #[inline(always)]
7186 fn expand_merge_u64x2(
7187 self,
7188 values: u64x2<Self>,
7189 mask: mask64x2<Self>,
7190 merge: u64x2<Self>,
7191 ) -> u64x2<Self> {
7192 crate::kernel!(
7193 #[inline(always)]
7194 fn kernel(
7195 token: Avx512,
7196 values: u64x2<Avx512>,
7197 mask: mask64x2<Avx512>,
7198 merge: u64x2<Avx512>,
7199 ) -> u64x2<Avx512> {
7200 _mm_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into())
7201 .simd_into(token)
7202 }
7203 );
7204 kernel(self, values, mask, merge)
7205 }
7206 #[inline(always)]
7207 fn count_ones_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
7208 crate::kernel!(
7209 #[inline(always)]
7210 fn kernel(token: Avx512, a: u64x2<Avx512>) -> u64x2<Avx512> {
7211 _mm_popcnt_epi64(a.into()).simd_into(token)
7212 }
7213 );
7214 kernel(self, a)
7215 }
7216 #[inline(always)]
7217 fn count_zeros_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
7218 self.count_ones_u64x2(self.not_u64x2(a))
7219 }
7220 #[inline(always)]
7221 fn add_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7222 crate::kernel!(
7223 #[inline(always)]
7224 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7225 _mm_add_epi64(a.into(), b.into()).simd_into(token)
7226 }
7227 );
7228 kernel(self, a, b)
7229 }
7230 #[inline(always)]
7231 fn saturating_add_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7232 crate::kernel!(
7233 #[inline(always)]
7234 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7235 let a = a.into();
7236 let b = b.into();
7237 let threshold = _mm_xor_si128(b, _mm_set1_epi64x(-1));
7238 _mm_add_epi64(_mm_min_epu64(a, threshold), b).simd_into(token)
7239 }
7240 );
7241 kernel(self, a, b)
7242 }
7243 #[inline(always)]
7244 fn sub_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7245 crate::kernel!(
7246 #[inline(always)]
7247 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7248 _mm_sub_epi64(a.into(), b.into()).simd_into(token)
7249 }
7250 );
7251 kernel(self, a, b)
7252 }
7253 #[inline(always)]
7254 fn saturating_sub_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7255 crate::kernel!(
7256 #[inline(always)]
7257 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7258 let a = a.into();
7259 let b = b.into();
7260 _mm_sub_epi64(_mm_max_epu64(a, b), b).simd_into(token)
7261 }
7262 );
7263 kernel(self, a, b)
7264 }
7265 #[inline(always)]
7266 fn mul_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7267 crate::kernel!(
7268 #[inline(always)]
7269 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7270 _mm_mullo_epi64(a.into(), b.into()).simd_into(token)
7271 }
7272 );
7273 kernel(self, a, b)
7274 }
7275 #[inline(always)]
7276 fn and_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7277 crate::kernel!(
7278 #[inline(always)]
7279 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7280 _mm_and_si128(a.into(), b.into()).simd_into(token)
7281 }
7282 );
7283 kernel(self, a, b)
7284 }
7285 #[inline(always)]
7286 fn or_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7287 crate::kernel!(
7288 #[inline(always)]
7289 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7290 _mm_or_si128(a.into(), b.into()).simd_into(token)
7291 }
7292 );
7293 kernel(self, a, b)
7294 }
7295 #[inline(always)]
7296 fn xor_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7297 crate::kernel!(
7298 #[inline(always)]
7299 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7300 _mm_xor_si128(a.into(), b.into()).simd_into(token)
7301 }
7302 );
7303 kernel(self, a, b)
7304 }
7305 #[inline(always)]
7306 fn not_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
7307 a ^ !0
7308 }
7309 #[inline(always)]
7310 fn shl_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
7311 crate::kernel!(
7312 #[inline(always)]
7313 fn kernel(token: Avx512, a: u64x2<Avx512>, shift: u32) -> u64x2<Avx512> {
7314 _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
7315 }
7316 );
7317 kernel(self, a, shift)
7318 }
7319 #[inline(always)]
7320 fn shlv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7321 crate::kernel!(
7322 #[inline(always)]
7323 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7324 _mm_sllv_epi64(a.into(), b.into()).simd_into(token)
7325 }
7326 );
7327 kernel(self, a, b)
7328 }
7329 #[inline(always)]
7330 fn shr_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
7331 crate::kernel!(
7332 #[inline(always)]
7333 fn kernel(token: Avx512, a: u64x2<Avx512>, shift: u32) -> u64x2<Avx512> {
7334 _mm_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
7335 }
7336 );
7337 kernel(self, a, shift)
7338 }
7339 #[inline(always)]
7340 fn shrv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7341 crate::kernel!(
7342 #[inline(always)]
7343 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7344 _mm_srlv_epi64(a.into(), b.into()).simd_into(token)
7345 }
7346 );
7347 kernel(self, a, b)
7348 }
7349 #[inline(always)]
7350 fn reduce_max_u64x2(self, a: u64x2<Self>) -> u64 {
7351 crate::kernel!(
7352 #[inline(always)]
7353 fn kernel(token: Avx512, a: u64x2<Avx512>) -> u64 {
7354 let lanes: [u64; 2] = a.into();
7355 lanes[0].max(lanes[1])
7356 }
7357 );
7358 kernel(self, a)
7359 }
7360 #[inline(always)]
7361 fn reduce_min_u64x2(self, a: u64x2<Self>) -> u64 {
7362 crate::kernel!(
7363 #[inline(always)]
7364 fn kernel(token: Avx512, a: u64x2<Avx512>) -> u64 {
7365 let lanes: [u64; 2] = a.into();
7366 lanes[0].min(lanes[1])
7367 }
7368 );
7369 kernel(self, a)
7370 }
7371 #[inline(always)]
7372 fn reduce_sum_u64x2(self, a: u64x2<Self>) -> u64 {
7373 crate::kernel!(
7374 #[inline(always)]
7375 fn kernel(token: Avx512, a: u64x2<Avx512>) -> u64 {
7376 let sum: __m128i = a.into();
7377 let sum = _mm_add_epi64(sum, _mm_srli_si128::<8>(sum));
7378 let lanes: [u64; 2usize] = crate::transmute::checked_transmute_copy(&sum);
7379 lanes[0]
7380 }
7381 );
7382 kernel(self, a)
7383 }
7384 #[inline(always)]
7385 fn reduce_product_u64x2(self, a: u64x2<Self>) -> u64 {
7386 let mul_level_0: [u64; 1usize] = [a[0usize].wrapping_mul(a[1usize])];
7387 mul_level_0[0]
7388 }
7389 #[inline(always)]
7390 fn max_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7391 crate::kernel!(
7392 #[inline(always)]
7393 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7394 _mm_max_epu64(a.into(), b.into()).simd_into(token)
7395 }
7396 );
7397 kernel(self, a, b)
7398 }
7399 #[inline(always)]
7400 fn min_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7401 crate::kernel!(
7402 #[inline(always)]
7403 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7404 _mm_min_epu64(a.into(), b.into()).simd_into(token)
7405 }
7406 );
7407 kernel(self, a, b)
7408 }
7409 #[inline(always)]
7410 fn clamp_u64x2(self, a: u64x2<Self>, b: u64x2<Self>, c: u64x2<Self>) -> u64x2<Self> {
7411 debug_assert!(
7412 b.simd_le(c).all_true(),
7413 "clamp requires min <= max for all elements"
7414 );
7415 a.max(b).min(c)
7416 }
7417 #[inline(always)]
7418 fn simd_eq_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7419 crate::kernel!(
7420 #[inline(always)]
7421 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> mask64x2<Avx512> {
7422 mask64x2 {
7423 val: _mm_cmpeq_epu64_mask(a.into(), b.into()),
7424 simd: token,
7425 }
7426 }
7427 );
7428 kernel(self, a, b)
7429 }
7430 #[inline(always)]
7431 fn simd_ne_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7432 crate::kernel!(
7433 #[inline(always)]
7434 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> mask64x2<Avx512> {
7435 mask64x2 {
7436 val: _mm_cmpneq_epu64_mask(a.into(), b.into()),
7437 simd: token,
7438 }
7439 }
7440 );
7441 kernel(self, a, b)
7442 }
7443 #[inline(always)]
7444 fn simd_lt_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7445 crate::kernel!(
7446 #[inline(always)]
7447 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> mask64x2<Avx512> {
7448 mask64x2 {
7449 val: _mm_cmplt_epu64_mask(a.into(), b.into()),
7450 simd: token,
7451 }
7452 }
7453 );
7454 kernel(self, a, b)
7455 }
7456 #[inline(always)]
7457 fn simd_le_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7458 crate::kernel!(
7459 #[inline(always)]
7460 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> mask64x2<Avx512> {
7461 mask64x2 {
7462 val: _mm_cmple_epu64_mask(a.into(), b.into()),
7463 simd: token,
7464 }
7465 }
7466 );
7467 kernel(self, a, b)
7468 }
7469 #[inline(always)]
7470 fn zip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7471 crate::kernel!(
7472 #[inline(always)]
7473 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7474 _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
7475 }
7476 );
7477 kernel(self, a, b)
7478 }
7479 #[inline(always)]
7480 fn zip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7481 crate::kernel!(
7482 #[inline(always)]
7483 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7484 _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
7485 }
7486 );
7487 kernel(self, a, b)
7488 }
7489 #[inline(always)]
7490 fn unzip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7491 crate::kernel!(
7492 #[inline(always)]
7493 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7494 _mm_permutex2var_epi64(a.into(), _mm_set_epi64x(2, 0), b.into()).simd_into(token)
7495 }
7496 );
7497 kernel(self, a, b)
7498 }
7499 #[inline(always)]
7500 fn unzip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7501 crate::kernel!(
7502 #[inline(always)]
7503 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x2<Avx512> {
7504 _mm_permutex2var_epi64(a.into(), _mm_set_epi64x(3, 1), b.into()).simd_into(token)
7505 }
7506 );
7507 kernel(self, a, b)
7508 }
7509 #[inline(always)]
7510 fn interleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
7511 (self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b))
7512 }
7513 #[inline(always)]
7514 fn deinterleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
7515 crate::kernel!(
7516 #[inline(always)]
7517 fn kernel(
7518 token: Avx512,
7519 a: u64x2<Avx512>,
7520 b: u64x2<Avx512>,
7521 ) -> (u64x2<Avx512>, u64x2<Avx512>) {
7522 let a = a.into();
7523 let b = b.into();
7524 (
7525 _mm_permutex2var_epi64(a, _mm_set_epi64x(2, 0), b).simd_into(token),
7526 _mm_permutex2var_epi64(a, _mm_set_epi64x(3, 1), b).simd_into(token),
7527 )
7528 }
7529 );
7530 kernel(self, a, b)
7531 }
7532 #[inline(always)]
7533 fn select_u64x2(self, a: mask64x2<Self>, b: u64x2<Self>, c: u64x2<Self>) -> u64x2<Self> {
7534 crate::kernel!(
7535 #[inline(always)]
7536 fn kernel(
7537 token: Avx512,
7538 a: mask64x2<Avx512>,
7539 b: u64x2<Avx512>,
7540 c: u64x2<Avx512>,
7541 ) -> u64x2<Avx512> {
7542 _mm_mask_blend_epi64(a.val, c.into(), b.into()).simd_into(token)
7543 }
7544 );
7545 kernel(self, a, b, c)
7546 }
7547 #[inline(always)]
7548 fn combine_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x4<Self> {
7549 crate::kernel!(
7550 #[inline(always)]
7551 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u64x4<Avx512> {
7552 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
7553 }
7554 );
7555 kernel(self, a, b)
7556 }
7557 #[inline(always)]
7558 fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2<Self>; 4usize] {
7559 crate::kernel!(
7560 #[inline(always)]
7561 fn kernel(token: Avx512, src: &[u64; 8usize]) -> [u64x2<Avx512>; 4usize] {
7562 let lanes: __m512i =
7563 crate::transmute::checked_transmute_copy::<[u64; 8usize], __m512i>(src);
7564 let lanes =
7565 _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes);
7566 let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes);
7567 [
7568 outputs[0].simd_into(token),
7569 outputs[1].simd_into(token),
7570 outputs[2].simd_into(token),
7571 outputs[3].simd_into(token),
7572 ]
7573 }
7574 );
7575 kernel(self, src)
7576 }
7577 #[inline(always)]
7578 fn store_four_interleaved_u64x2(
7579 self,
7580 vectors: [u64x2<Self>; 4usize],
7581 dest: &mut [u64; 8usize],
7582 ) -> () {
7583 crate::kernel!(
7584 #[inline(always)]
7585 fn kernel(
7586 token: Avx512,
7587 vectors: [u64x2<Avx512>; 4usize],
7588 dest: &mut [u64; 8usize],
7589 ) -> () {
7590 let _ = token;
7591 let inputs: [__m128i; 4] = [
7592 vectors[0].into(),
7593 vectors[1].into(),
7594 vectors[2].into(),
7595 vectors[3].into(),
7596 ];
7597 let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs);
7598 let lanes =
7599 _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), lanes);
7600 crate::transmute::checked_transmute_store::<__m512i, [u64; 8usize]>(lanes, dest);
7601 }
7602 );
7603 kernel(self, vectors, dest);
7604 }
7605 #[inline(always)]
7606 fn narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7607 crate::kernel!(
7608 #[inline(always)]
7609 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u32x4<Avx512> {
7610 let low = _mm_cvtepi64_epi32(a.into());
7611 let high = _mm_cvtepi64_epi32(b.into());
7612 _mm_unpacklo_epi64(low, high).simd_into(token)
7613 }
7614 );
7615 kernel(self, a, b)
7616 }
7617 #[inline(always)]
7618 fn saturating_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7619 crate::kernel!(
7620 #[inline(always)]
7621 fn kernel(token: Avx512, a: u64x2<Avx512>, b: u64x2<Avx512>) -> u32x4<Avx512> {
7622 let a = a.into();
7623 let b = b.into();
7624 let low = _mm_castps_si128(_mm_shuffle_ps::<0x88>(
7625 _mm_castsi128_ps(a),
7626 _mm_castsi128_ps(b),
7627 ));
7628 let high = _mm_castps_si128(_mm_shuffle_ps::<0xdd>(
7629 _mm_castsi128_ps(a),
7630 _mm_castsi128_ps(b),
7631 ));
7632 let zero = _mm_setzero_si128();
7633 let fits = _mm_cmpeq_epi32(high, zero);
7634 let ones = _mm_cmpeq_epi32(zero, zero);
7635 _mm_or_si128(low, _mm_xor_si128(fits, ones)).simd_into(token)
7636 }
7637 );
7638 kernel(self, a, b)
7639 }
7640 #[inline(always)]
7641 fn relaxed_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7642 debug_assert!(
7643 a.as_slice()
7644 .iter()
7645 .chain(b.as_slice())
7646 .all(|&value| { value >= u32::MIN as u64 && value <= u32::MAX as u64 }),
7647 "relaxed_narrow inputs must fit in the destination type",
7648 );
7649 self.narrow_u64x2(a, b)
7650 }
7651 #[inline(always)]
7652 fn cvt_f64_u64x2(self, a: u64x2<Self>) -> f64x2<Self> {
7653 crate::kernel!(
7654 #[inline(always)]
7655 fn kernel(token: Avx512, a: u64x2<Avx512>) -> f64x2<Avx512> {
7656 _mm_cvtepu64_pd(a.into()).simd_into(token)
7657 }
7658 );
7659 kernel(self, a)
7660 }
7661 #[inline(always)]
7662 fn splat_mask64x2(self, val: bool) -> mask64x2<Self> {
7663 mask64x2 {
7664 val: (if val { 3u64 } else { 0 }) as _,
7665 simd: self,
7666 }
7667 }
7668 #[inline(always)]
7669 fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2<Self> {
7670 mask64x2 {
7671 val: (bits & 3u64) as _,
7672 simd: self,
7673 }
7674 }
7675 #[inline(always)]
7676 fn to_bitmask_mask64x2(self, a: mask64x2<Self>) -> u64 {
7677 u64::from((a).val) & 3u64
7678 }
7679 #[inline(always)]
7680 fn from_vector_mask64x2(self, vector: i64x2<Self>) -> mask64x2<Self> {
7681 crate::kernel!(
7682 #[inline(always)]
7683 fn kernel(token: Avx512, vector: i64x2<Avx512>) -> mask64x2<Avx512> {
7684 _mm_movepi64_mask(vector.into()).simd_into(token)
7685 }
7686 );
7687 kernel(self, vector)
7688 }
7689 #[inline(always)]
7690 fn to_vector_mask64x2(self, a: mask64x2<Self>) -> i64x2<Self> {
7691 crate::kernel!(
7692 #[inline(always)]
7693 fn kernel(token: Avx512, a: mask64x2<Avx512>) -> i64x2<Avx512> {
7694 _mm_movm_epi64(a.val).simd_into(token)
7695 }
7696 );
7697 kernel(self, a)
7698 }
7699 #[inline(always)]
7700 fn set_mask64x2(self, a: &mut mask64x2<Self>, index: usize, value: bool) -> () {
7701 assert!(
7702 index < 2usize,
7703 "mask lane index {index} is out of bounds for {} lanes",
7704 2usize
7705 );
7706 let bit = 1u64 << index;
7707 let bits = u64::from((a).val);
7708 let bits = if value { bits | bit } else { bits & !bit };
7709 *a = mask64x2 {
7710 val: (bits) as _,
7711 simd: self,
7712 };
7713 }
7714 #[inline(always)]
7715 fn rotate_elements_left_mask64x2<const OFFSET: usize>(
7716 self,
7717 a: mask64x2<Self>,
7718 ) -> mask64x2<Self> {
7719 let bits = u64::from((a).val) & 3u64;
7720 let offset = OFFSET % 2;
7721 let bits = if offset == 0 {
7722 bits
7723 } else {
7724 ((bits >> offset) | (bits << (2 - offset))) & 3u64
7725 };
7726 mask64x2 {
7727 val: (bits) as _,
7728 simd: self,
7729 }
7730 }
7731 #[inline(always)]
7732 fn rotate_elements_right_mask64x2<const OFFSET: usize>(
7733 self,
7734 a: mask64x2<Self>,
7735 ) -> mask64x2<Self> {
7736 let bits = u64::from((a).val) & 3u64;
7737 let offset = OFFSET % 2;
7738 let bits = if offset == 0 {
7739 bits
7740 } else {
7741 ((bits << offset) | (bits >> (2 - offset))) & 3u64
7742 };
7743 mask64x2 {
7744 val: (bits) as _,
7745 simd: self,
7746 }
7747 }
7748 #[inline(always)]
7749 fn and_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7750 mask64x2 {
7751 val: ((u64::from((a).val) & u64::from((b).val)) & 3u64) as _,
7752 simd: self,
7753 }
7754 }
7755 #[inline(always)]
7756 fn or_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7757 mask64x2 {
7758 val: ((u64::from((a).val) | u64::from((b).val)) & 3u64) as _,
7759 simd: self,
7760 }
7761 }
7762 #[inline(always)]
7763 fn xor_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7764 mask64x2 {
7765 val: ((u64::from((a).val) ^ u64::from((b).val)) & 3u64) as _,
7766 simd: self,
7767 }
7768 }
7769 #[inline(always)]
7770 fn not_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
7771 mask64x2 {
7772 val: ((!u64::from((a).val)) & 3u64) as _,
7773 simd: self,
7774 }
7775 }
7776 #[inline(always)]
7777 fn reverse_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
7778 mask64x2 {
7779 val: (a.val.reverse_bits() >> 6usize) as _,
7780 simd: self,
7781 }
7782 }
7783 #[inline(always)]
7784 fn select_mask64x2(
7785 self,
7786 a: mask64x2<Self>,
7787 b: mask64x2<Self>,
7788 c: mask64x2<Self>,
7789 ) -> mask64x2<Self> {
7790 mask64x2 {
7791 val: (((u64::from((a).val) & u64::from((b).val))
7792 | ((!u64::from((a).val)) & u64::from((c).val)))
7793 & 3u64) as _,
7794 simd: self,
7795 }
7796 }
7797 #[inline(always)]
7798 fn simd_eq_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7799 mask64x2 {
7800 val: (!u64::from(a.val ^ b.val) & 3u64) as _,
7801 simd: self,
7802 }
7803 }
7804 #[inline(always)]
7805 fn simd_ne_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7806 mask64x2 {
7807 val: (u64::from(a.val ^ b.val) & 3u64) as _,
7808 simd: self,
7809 }
7810 }
7811 #[inline(always)]
7812 fn any_true_mask64x2(self, a: mask64x2<Self>) -> bool {
7813 let bits = u64::from((a).val) & 3u64;
7814 bits != 0
7815 }
7816 #[inline(always)]
7817 fn all_true_mask64x2(self, a: mask64x2<Self>) -> bool {
7818 let bits = u64::from((a).val) & 3u64;
7819 bits == 3u64
7820 }
7821 #[inline(always)]
7822 fn any_false_mask64x2(self, a: mask64x2<Self>) -> bool {
7823 let bits = u64::from((a).val) & 3u64;
7824 bits != 3u64
7825 }
7826 #[inline(always)]
7827 fn all_false_mask64x2(self, a: mask64x2<Self>) -> bool {
7828 let bits = u64::from((a).val) & 3u64;
7829 bits == 0
7830 }
7831 #[inline(always)]
7832 fn combine_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x4<Self> {
7833 let bits = (u64::from(a.val) | (u64::from(b.val) << 2usize)) & 15u64;
7834 mask64x4 {
7835 val: bits as _,
7836 simd: self,
7837 }
7838 }
7839 #[inline(always)]
7840 fn narrow_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask32x4<Self> {
7841 let low = u64::from(a.val) & 3u64;
7842 let high = u64::from(b.val) & 3u64;
7843 mask32x4 {
7844 val: (low | (high << 2usize)) as _,
7845 simd: self,
7846 }
7847 }
7848 #[inline(always)]
7849 fn abs_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
7850 crate::kernel!(
7851 #[inline(always)]
7852 fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
7853 _mm256_andnot_ps(_mm256_set1_ps(-0.0), a.into()).simd_into(token)
7854 }
7855 );
7856 kernel(self, a)
7857 }
7858 #[inline(always)]
7859 fn splat_f32x8(self, val: f32) -> f32x8<Self> {
7860 crate::kernel!(
7861 #[inline(always)]
7862 fn kernel(token: Avx512, val: f32) -> f32x8<Avx512> {
7863 _mm256_set1_ps(val).simd_into(token)
7864 }
7865 );
7866 kernel(self, val)
7867 }
7868 #[inline(always)]
7869 fn reverse_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
7870 let indices: u8x32<Self> = [
7871 28, 29, 30, 31, 24, 25, 26, 27, 20, 21, 22, 23, 16, 17, 18, 19, 12, 13, 14, 15, 8, 9,
7872 10, 11, 4, 5, 6, 7, 0, 1, 2, 3,
7873 ]
7874 .simd_into(self);
7875 self.swizzle_dyn_f32x8(a, indices)
7876 }
7877 #[inline(always)]
7878 fn slide_f32x8<const SHIFT: usize>(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7879 crate::kernel!(
7880 #[inline(always)]
7881 fn kernel(
7882 token: Avx512,
7883 a: f32x8<Avx512>,
7884 b: f32x8<Avx512>,
7885 shift: usize,
7886 ) -> f32x8<Avx512> {
7887 if shift >= 8usize {
7888 return b;
7889 }
7890 let idx = _mm256_add_epi8(
7891 _mm256_setr_epi8(
7892 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
7893 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
7894 ),
7895 _mm256_set1_epi8((shift * 4usize) as i8),
7896 );
7897 let result = _mm256_permutex2var_epi8(
7898 Bytes::to_bytes(a).val.0,
7899 idx,
7900 Bytes::to_bytes(b).val.0,
7901 );
7902 Bytes::from_bytes(u8x32 {
7903 val: crate::support::Aligned256(result),
7904 simd: token,
7905 })
7906 }
7907 );
7908 kernel(self, a, b, SHIFT)
7909 }
7910 #[inline(always)]
7911 fn slide_within_blocks_f32x8<const SHIFT: usize>(
7912 self,
7913 a: f32x8<Self>,
7914 b: f32x8<Self>,
7915 ) -> f32x8<Self> {
7916 if SHIFT == 0 {
7917 return a;
7918 }
7919 if SHIFT >= 4usize {
7920 return b;
7921 }
7922 let a = Bytes::to_bytes(a).val.0;
7923 let b = Bytes::to_bytes(b).val.0;
7924 let result = dyn_alignr_256(self, b, a, SHIFT * 4usize);
7925 Bytes::from_bytes(u8x32 {
7926 val: crate::support::Aligned256(result),
7927 simd: self,
7928 })
7929 }
7930 #[inline(always)]
7931 fn compress_f32x8(self, values: f32x8<Self>, mask: mask32x8<Self>) -> f32x8<Self> {
7932 crate::kernel!(
7933 #[inline(always)]
7934 fn kernel(
7935 token: Avx512,
7936 values: f32x8<Avx512>,
7937 mask: mask32x8<Avx512>,
7938 ) -> f32x8<Avx512> {
7939 _mm256_maskz_compress_ps(u64::from((mask).val) as u8, values.into())
7940 .simd_into(token)
7941 }
7942 );
7943 kernel(self, values, mask)
7944 }
7945 #[inline(always)]
7946 fn compress_merge_f32x8(
7947 self,
7948 values: f32x8<Self>,
7949 mask: mask32x8<Self>,
7950 merge: f32x8<Self>,
7951 ) -> f32x8<Self> {
7952 crate::kernel!(
7953 #[inline(always)]
7954 fn kernel(
7955 token: Avx512,
7956 values: f32x8<Avx512>,
7957 mask: mask32x8<Avx512>,
7958 merge: f32x8<Avx512>,
7959 ) -> f32x8<Avx512> {
7960 _mm256_mask_compress_ps(merge.into(), u64::from((mask).val) as u8, values.into())
7961 .simd_into(token)
7962 }
7963 );
7964 kernel(self, values, mask, merge)
7965 }
7966 #[inline(always)]
7967 fn expand_f32x8(self, values: f32x8<Self>, mask: mask32x8<Self>) -> f32x8<Self> {
7968 crate::kernel!(
7969 #[inline(always)]
7970 fn kernel(
7971 token: Avx512,
7972 values: f32x8<Avx512>,
7973 mask: mask32x8<Avx512>,
7974 ) -> f32x8<Avx512> {
7975 _mm256_maskz_expand_ps(u64::from((mask).val) as u8, values.into()).simd_into(token)
7976 }
7977 );
7978 kernel(self, values, mask)
7979 }
7980 #[inline(always)]
7981 fn expand_merge_f32x8(
7982 self,
7983 values: f32x8<Self>,
7984 mask: mask32x8<Self>,
7985 merge: f32x8<Self>,
7986 ) -> f32x8<Self> {
7987 crate::kernel!(
7988 #[inline(always)]
7989 fn kernel(
7990 token: Avx512,
7991 values: f32x8<Avx512>,
7992 mask: mask32x8<Avx512>,
7993 merge: f32x8<Avx512>,
7994 ) -> f32x8<Avx512> {
7995 _mm256_mask_expand_ps(merge.into(), u64::from((mask).val) as u8, values.into())
7996 .simd_into(token)
7997 }
7998 );
7999 kernel(self, values, mask, merge)
8000 }
8001 #[inline(always)]
8002 fn neg_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8003 crate::kernel!(
8004 #[inline(always)]
8005 fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
8006 _mm256_xor_ps(a.into(), _mm256_set1_ps(-0.0)).simd_into(token)
8007 }
8008 );
8009 kernel(self, a)
8010 }
8011 #[inline(always)]
8012 fn sqrt_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8013 crate::kernel!(
8014 #[inline(always)]
8015 fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
8016 _mm256_sqrt_ps(a.into()).simd_into(token)
8017 }
8018 );
8019 kernel(self, a)
8020 }
8021 #[inline(always)]
8022 fn approximate_recip_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8023 crate::kernel!(
8024 #[inline(always)]
8025 fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
8026 _mm256_rcp14_ps(a.into()).simd_into(token)
8027 }
8028 );
8029 kernel(self, a)
8030 }
8031 #[inline(always)]
8032 fn add_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8033 crate::kernel!(
8034 #[inline(always)]
8035 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
8036 _mm256_add_ps(a.into(), b.into()).simd_into(token)
8037 }
8038 );
8039 kernel(self, a, b)
8040 }
8041 #[inline(always)]
8042 fn sub_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8043 crate::kernel!(
8044 #[inline(always)]
8045 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
8046 _mm256_sub_ps(a.into(), b.into()).simd_into(token)
8047 }
8048 );
8049 kernel(self, a, b)
8050 }
8051 #[inline(always)]
8052 fn mul_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8053 crate::kernel!(
8054 #[inline(always)]
8055 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
8056 _mm256_mul_ps(a.into(), b.into()).simd_into(token)
8057 }
8058 );
8059 kernel(self, a, b)
8060 }
8061 #[inline(always)]
8062 fn div_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8063 crate::kernel!(
8064 #[inline(always)]
8065 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
8066 _mm256_div_ps(a.into(), b.into()).simd_into(token)
8067 }
8068 );
8069 kernel(self, a, b)
8070 }
8071 #[inline(always)]
8072 fn copysign_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8073 crate::kernel!(
8074 #[inline(always)]
8075 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
8076 let mask = _mm256_set1_ps(-0.0);
8077 _mm256_or_ps(
8078 _mm256_and_ps(mask, b.into()),
8079 _mm256_andnot_ps(mask, a.into()),
8080 )
8081 .simd_into(token)
8082 }
8083 );
8084 kernel(self, a, b)
8085 }
8086 #[inline(always)]
8087 fn max_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8088 crate::kernel!(
8089 #[inline(always)]
8090 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
8091 _mm256_max_ps(a.into(), b.into()).simd_into(token)
8092 }
8093 );
8094 kernel(self, a, b)
8095 }
8096 #[inline(always)]
8097 fn min_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8098 crate::kernel!(
8099 #[inline(always)]
8100 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
8101 _mm256_min_ps(a.into(), b.into()).simd_into(token)
8102 }
8103 );
8104 kernel(self, a, b)
8105 }
8106 #[inline(always)]
8107 fn max_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8108 crate::kernel!(
8109 #[inline(always)]
8110 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
8111 _mm256_range_ps::<5i32>(a.into(), b.into()).simd_into(token)
8112 }
8113 );
8114 kernel(self, a, b)
8115 }
8116 #[inline(always)]
8117 fn min_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8118 crate::kernel!(
8119 #[inline(always)]
8120 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
8121 _mm256_range_ps::<4i32>(a.into(), b.into()).simd_into(token)
8122 }
8123 );
8124 kernel(self, a, b)
8125 }
8126 #[inline(always)]
8127 fn clamp_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
8128 crate::kernel!(
8129 #[inline(always)]
8130 fn kernel(
8131 token: Avx512,
8132 a: f32x8<Avx512>,
8133 b: f32x8<Avx512>,
8134 c: f32x8<Avx512>,
8135 ) -> f32x8<Avx512> {
8136 debug_assert!(
8137 b.simd_le(c).all_true(),
8138 "clamp requires min <= max for all elements"
8139 );
8140 _mm256_min_ps(c.into(), _mm256_max_ps(b.into(), a.into())).simd_into(token)
8141 }
8142 );
8143 kernel(self, a, b, c)
8144 }
8145 #[inline(always)]
8146 fn simd_eq_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
8147 crate::kernel!(
8148 #[inline(always)]
8149 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> mask32x8<Avx512> {
8150 mask32x8 {
8151 val: _mm256_cmp_ps_mask::<0i32>(a.into(), b.into()),
8152 simd: token,
8153 }
8154 }
8155 );
8156 kernel(self, a, b)
8157 }
8158 #[inline(always)]
8159 fn simd_ne_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
8160 crate::kernel!(
8161 #[inline(always)]
8162 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> mask32x8<Avx512> {
8163 mask32x8 {
8164 val: _mm256_cmp_ps_mask::<4i32>(a.into(), b.into()),
8165 simd: token,
8166 }
8167 }
8168 );
8169 kernel(self, a, b)
8170 }
8171 #[inline(always)]
8172 fn simd_lt_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
8173 crate::kernel!(
8174 #[inline(always)]
8175 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> mask32x8<Avx512> {
8176 mask32x8 {
8177 val: _mm256_cmp_ps_mask::<17i32>(a.into(), b.into()),
8178 simd: token,
8179 }
8180 }
8181 );
8182 kernel(self, a, b)
8183 }
8184 #[inline(always)]
8185 fn simd_le_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
8186 crate::kernel!(
8187 #[inline(always)]
8188 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> mask32x8<Avx512> {
8189 mask32x8 {
8190 val: _mm256_cmp_ps_mask::<18i32>(a.into(), b.into()),
8191 simd: token,
8192 }
8193 }
8194 );
8195 kernel(self, a, b)
8196 }
8197 #[inline(always)]
8198 fn zip_low_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8199 crate::kernel!(
8200 #[inline(always)]
8201 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
8202 _mm256_permutex2var_ps(
8203 a.into(),
8204 _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11),
8205 b.into(),
8206 )
8207 .simd_into(token)
8208 }
8209 );
8210 kernel(self, a, b)
8211 }
8212 #[inline(always)]
8213 fn zip_high_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8214 crate::kernel!(
8215 #[inline(always)]
8216 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
8217 _mm256_permutex2var_ps(
8218 a.into(),
8219 _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15),
8220 b.into(),
8221 )
8222 .simd_into(token)
8223 }
8224 );
8225 kernel(self, a, b)
8226 }
8227 #[inline(always)]
8228 fn unzip_low_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8229 crate::kernel!(
8230 #[inline(always)]
8231 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
8232 _mm256_permutex2var_ps(
8233 a.into(),
8234 _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14),
8235 b.into(),
8236 )
8237 .simd_into(token)
8238 }
8239 );
8240 kernel(self, a, b)
8241 }
8242 #[inline(always)]
8243 fn unzip_high_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8244 crate::kernel!(
8245 #[inline(always)]
8246 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
8247 _mm256_permutex2var_ps(
8248 a.into(),
8249 _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15),
8250 b.into(),
8251 )
8252 .simd_into(token)
8253 }
8254 );
8255 kernel(self, a, b)
8256 }
8257 #[inline(always)]
8258 fn interleave_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> (f32x8<Self>, f32x8<Self>) {
8259 crate::kernel!(
8260 #[inline(always)]
8261 fn kernel(
8262 token: Avx512,
8263 a: f32x8<Avx512>,
8264 b: f32x8<Avx512>,
8265 ) -> (f32x8<Avx512>, f32x8<Avx512>) {
8266 let a = a.into();
8267 let b = b.into();
8268 (
8269 _mm256_permutex2var_ps(a, _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11), b)
8270 .simd_into(token),
8271 _mm256_permutex2var_ps(a, _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15), b)
8272 .simd_into(token),
8273 )
8274 }
8275 );
8276 kernel(self, a, b)
8277 }
8278 #[inline(always)]
8279 fn deinterleave_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> (f32x8<Self>, f32x8<Self>) {
8280 crate::kernel!(
8281 #[inline(always)]
8282 fn kernel(
8283 token: Avx512,
8284 a: f32x8<Avx512>,
8285 b: f32x8<Avx512>,
8286 ) -> (f32x8<Avx512>, f32x8<Avx512>) {
8287 let a = a.into();
8288 let b = b.into();
8289 (
8290 _mm256_permutex2var_ps(a, _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14), b)
8291 .simd_into(token),
8292 _mm256_permutex2var_ps(a, _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15), b)
8293 .simd_into(token),
8294 )
8295 }
8296 );
8297 kernel(self, a, b)
8298 }
8299 #[inline(always)]
8300 fn mul_add_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
8301 crate::kernel!(
8302 #[inline(always)]
8303 fn kernel(
8304 token: Avx512,
8305 a: f32x8<Avx512>,
8306 b: f32x8<Avx512>,
8307 c: f32x8<Avx512>,
8308 ) -> f32x8<Avx512> {
8309 _mm256_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token)
8310 }
8311 );
8312 kernel(self, a, b, c)
8313 }
8314 #[inline(always)]
8315 fn mul_add_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
8316 self.mul_add_f32x8(a, b, c)
8317 }
8318 #[inline(always)]
8319 fn mul_sub_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
8320 crate::kernel!(
8321 #[inline(always)]
8322 fn kernel(
8323 token: Avx512,
8324 a: f32x8<Avx512>,
8325 b: f32x8<Avx512>,
8326 c: f32x8<Avx512>,
8327 ) -> f32x8<Avx512> {
8328 _mm256_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token)
8329 }
8330 );
8331 kernel(self, a, b, c)
8332 }
8333 #[inline(always)]
8334 fn mul_sub_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
8335 self.mul_sub_f32x8(a, b, c)
8336 }
8337 #[inline(always)]
8338 fn floor_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8339 crate::kernel!(
8340 #[inline(always)]
8341 fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
8342 _mm256_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
8343 .simd_into(token)
8344 }
8345 );
8346 kernel(self, a)
8347 }
8348 #[inline(always)]
8349 fn ceil_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8350 crate::kernel!(
8351 #[inline(always)]
8352 fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
8353 _mm256_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
8354 .simd_into(token)
8355 }
8356 );
8357 kernel(self, a)
8358 }
8359 #[inline(always)]
8360 fn round_ties_even_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8361 crate::kernel!(
8362 #[inline(always)]
8363 fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
8364 _mm256_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
8365 .simd_into(token)
8366 }
8367 );
8368 kernel(self, a)
8369 }
8370 #[inline(always)]
8371 fn round_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8372 let bias = self.splat_f32x8(const { f32::next_down(0.5) });
8373 (a + self.copysign_f32x8(bias, a)).trunc()
8374 }
8375 #[inline(always)]
8376 fn fract_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8377 a - self.trunc_f32x8(a)
8378 }
8379 #[inline(always)]
8380 fn trunc_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8381 crate::kernel!(
8382 #[inline(always)]
8383 fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
8384 _mm256_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
8385 .simd_into(token)
8386 }
8387 );
8388 kernel(self, a)
8389 }
8390 #[inline(always)]
8391 fn to_degrees_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8392 a * const { f32::to_degrees(1.0) }
8393 }
8394 #[inline(always)]
8395 fn to_radians_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8396 a * const { f32::to_radians(1.0) }
8397 }
8398 #[inline(always)]
8399 fn is_nan_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8400 crate::kernel!(
8401 #[inline(always)]
8402 fn kernel(token: Avx512, a: f32x8<Avx512>) -> mask32x8<Avx512> {
8403 mask32x8 {
8404 val: _mm256_fpclass_ps_mask::<129i32>(a.into()),
8405 simd: token,
8406 }
8407 }
8408 );
8409 kernel(self, a)
8410 }
8411 #[inline(always)]
8412 fn is_infinite_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8413 crate::kernel!(
8414 #[inline(always)]
8415 fn kernel(token: Avx512, a: f32x8<Avx512>) -> mask32x8<Avx512> {
8416 mask32x8 {
8417 val: _mm256_fpclass_ps_mask::<24i32>(a.into()),
8418 simd: token,
8419 }
8420 }
8421 );
8422 kernel(self, a)
8423 }
8424 #[inline(always)]
8425 fn is_finite_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8426 crate::kernel!(
8427 #[inline(always)]
8428 fn kernel(token: Avx512, a: f32x8<Avx512>) -> mask32x8<Avx512> {
8429 !(mask32x8 {
8430 val: _mm256_fpclass_ps_mask::<153i32>(a.into()),
8431 simd: token,
8432 })
8433 }
8434 );
8435 kernel(self, a)
8436 }
8437 #[inline(always)]
8438 fn is_subnormal_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8439 crate::kernel!(
8440 #[inline(always)]
8441 fn kernel(token: Avx512, a: f32x8<Avx512>) -> mask32x8<Avx512> {
8442 mask32x8 {
8443 val: _mm256_fpclass_ps_mask::<129i32>(_mm256_xor_ps(
8444 a.into(),
8445 _mm256_set1_ps(f32::INFINITY),
8446 )),
8447 simd: token,
8448 }
8449 }
8450 );
8451 kernel(self, a)
8452 }
8453 #[inline(always)]
8454 fn is_normal_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8455 crate::kernel!(
8456 #[inline(always)]
8457 fn kernel(token: Avx512, a: f32x8<Avx512>) -> mask32x8<Avx512> {
8458 !(mask32x8 {
8459 val: _mm256_fpclass_ps_mask::<191i32>(a.into()),
8460 simd: token,
8461 })
8462 }
8463 );
8464 kernel(self, a)
8465 }
8466 #[inline(always)]
8467 fn is_sign_positive_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8468 let i: i32x8<Self> = a.bitcast();
8469 (i & i32::MIN).simd_eq(0)
8470 }
8471 #[inline(always)]
8472 fn is_sign_negative_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8473 !a.is_sign_positive()
8474 }
8475 #[inline(always)]
8476 fn select_f32x8(self, a: mask32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
8477 crate::kernel!(
8478 #[inline(always)]
8479 fn kernel(
8480 token: Avx512,
8481 a: mask32x8<Avx512>,
8482 b: f32x8<Avx512>,
8483 c: f32x8<Avx512>,
8484 ) -> f32x8<Avx512> {
8485 _mm256_mask_blend_ps(a.val, c.into(), b.into()).simd_into(token)
8486 }
8487 );
8488 kernel(self, a, b, c)
8489 }
8490 #[inline(always)]
8491 fn combine_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x16<Self> {
8492 crate::kernel!(
8493 #[inline(always)]
8494 fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x16<Avx512> {
8495 _mm512_insertf32x8::<1>(_mm512_castps256_ps512(a.into()), b.into()).simd_into(token)
8496 }
8497 );
8498 kernel(self, a, b)
8499 }
8500 #[inline(always)]
8501 fn split_f32x8(self, a: f32x8<Self>) -> (f32x4<Self>, f32x4<Self>) {
8502 crate::kernel!(
8503 #[inline(always)]
8504 fn kernel(token: Avx512, a: f32x8<Avx512>) -> (f32x4<Avx512>, f32x4<Avx512>) {
8505 (
8506 _mm256_extractf128_ps::<0>(a.into()).simd_into(token),
8507 _mm256_extractf128_ps::<1>(a.into()).simd_into(token),
8508 )
8509 }
8510 );
8511 kernel(self, a)
8512 }
8513 #[inline(always)]
8514 fn widen_f32x8(self, a: f32x8<Self>) -> (f64x4<Self>, f64x4<Self>) {
8515 crate::kernel!(
8516 #[inline(always)]
8517 fn kernel(token: Avx512, a: f32x8<Avx512>) -> (f64x4<Avx512>, f64x4<Avx512>) {
8518 let raw = a.into();
8519 (
8520 _mm256_cvtps_pd(_mm256_castps256_ps128(raw)).simd_into(token),
8521 _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(raw)).simd_into(token),
8522 )
8523 }
8524 );
8525 kernel(self, a)
8526 }
8527 #[inline(always)]
8528 fn cvt_u32_f32x8(self, a: f32x8<Self>) -> u32x8<Self> {
8529 crate::kernel!(
8530 #[inline(always)]
8531 fn kernel(token: Avx512, a: f32x8<Avx512>) -> u32x8<Avx512> {
8532 _mm256_cvttps_epu32(a.into()).simd_into(token)
8533 }
8534 );
8535 kernel(self, a)
8536 }
8537 #[inline(always)]
8538 fn cvt_u32_precise_f32x8(self, a: f32x8<Self>) -> u32x8<Self> {
8539 crate::kernel!(
8540 #[inline(always)]
8541 fn kernel(token: Avx512, a: f32x8<Avx512>) -> u32x8<Avx512> {
8542 let a = a.into();
8543 let positive = _mm256_cmp_ps_mask::<17i32>(_mm256_setzero_ps(), a);
8544 _mm256_maskz_cvttps_epu32(positive, a).simd_into(token)
8545 }
8546 );
8547 kernel(self, a)
8548 }
8549 #[inline(always)]
8550 fn cvt_i32_f32x8(self, a: f32x8<Self>) -> i32x8<Self> {
8551 crate::kernel!(
8552 #[inline(always)]
8553 fn kernel(token: Avx512, a: f32x8<Avx512>) -> i32x8<Avx512> {
8554 _mm256_cvttps_epi32(a.into()).simd_into(token)
8555 }
8556 );
8557 kernel(self, a)
8558 }
8559 #[inline(always)]
8560 fn cvt_i32_precise_f32x8(self, a: f32x8<Self>) -> i32x8<Self> {
8561 crate::kernel!(
8562 #[inline(always)]
8563 fn kernel(token: Avx512, a: f32x8<Avx512>) -> i32x8<Avx512> {
8564 let a = a.into();
8565 let in_range = _mm256_cmp_ps_mask::<17i32>(a, _mm256_set1_ps(2147483648.0));
8566 let mut converted =
8567 _mm256_mask_cvttps_epi32(_mm256_set1_epi32(i32::MAX), in_range, a);
8568 let is_not_nan = _mm256_cmp_ps_mask::<7i32>(a, a);
8569 converted = _mm256_mask_blend_epi32(is_not_nan, _mm256_setzero_si256(), converted);
8570 converted.simd_into(token)
8571 }
8572 );
8573 kernel(self, a)
8574 }
8575 #[inline(always)]
8576 fn abs_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
8577 crate::kernel!(
8578 #[inline(always)]
8579 fn kernel(token: Avx512, a: i8x32<Avx512>) -> i8x32<Avx512> {
8580 _mm256_abs_epi8(a.into()).simd_into(token)
8581 }
8582 );
8583 kernel(self, a)
8584 }
8585 #[inline(always)]
8586 fn splat_i8x32(self, val: i8) -> i8x32<Self> {
8587 crate::kernel!(
8588 #[inline(always)]
8589 fn kernel(token: Avx512, val: i8) -> i8x32<Avx512> {
8590 _mm256_set1_epi8(val).simd_into(token)
8591 }
8592 );
8593 kernel(self, val)
8594 }
8595 #[inline(always)]
8596 fn reverse_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
8597 let indices: u8x32<Self> = [
8598 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10,
8599 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
8600 ]
8601 .simd_into(self);
8602 self.swizzle_dyn_i8x32(a, indices)
8603 }
8604 #[inline(always)]
8605 fn slide_i8x32<const SHIFT: usize>(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8606 crate::kernel!(
8607 #[inline(always)]
8608 fn kernel(
8609 token: Avx512,
8610 a: i8x32<Avx512>,
8611 b: i8x32<Avx512>,
8612 shift: usize,
8613 ) -> i8x32<Avx512> {
8614 if shift >= 32usize {
8615 return b;
8616 }
8617 let idx = _mm256_add_epi8(
8618 _mm256_setr_epi8(
8619 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
8620 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
8621 ),
8622 _mm256_set1_epi8((shift) as i8),
8623 );
8624 let result = _mm256_permutex2var_epi8(
8625 Bytes::to_bytes(a).val.0,
8626 idx,
8627 Bytes::to_bytes(b).val.0,
8628 );
8629 Bytes::from_bytes(u8x32 {
8630 val: crate::support::Aligned256(result),
8631 simd: token,
8632 })
8633 }
8634 );
8635 kernel(self, a, b, SHIFT)
8636 }
8637 #[inline(always)]
8638 fn slide_within_blocks_i8x32<const SHIFT: usize>(
8639 self,
8640 a: i8x32<Self>,
8641 b: i8x32<Self>,
8642 ) -> i8x32<Self> {
8643 if SHIFT == 0 {
8644 return a;
8645 }
8646 if SHIFT >= 16usize {
8647 return b;
8648 }
8649 let a = Bytes::to_bytes(a).val.0;
8650 let b = Bytes::to_bytes(b).val.0;
8651 let result = dyn_alignr_256(self, b, a, SHIFT);
8652 Bytes::from_bytes(u8x32 {
8653 val: crate::support::Aligned256(result),
8654 simd: self,
8655 })
8656 }
8657 #[inline(always)]
8658 fn compress_i8x32(self, values: i8x32<Self>, mask: mask8x32<Self>) -> i8x32<Self> {
8659 crate::kernel!(
8660 #[inline(always)]
8661 fn kernel(
8662 token: Avx512,
8663 values: i8x32<Avx512>,
8664 mask: mask8x32<Avx512>,
8665 ) -> i8x32<Avx512> {
8666 _mm256_maskz_compress_epi8(u64::from((mask).val) as u32, values.into())
8667 .simd_into(token)
8668 }
8669 );
8670 kernel(self, values, mask)
8671 }
8672 #[inline(always)]
8673 fn compress_merge_i8x32(
8674 self,
8675 values: i8x32<Self>,
8676 mask: mask8x32<Self>,
8677 merge: i8x32<Self>,
8678 ) -> i8x32<Self> {
8679 crate::kernel!(
8680 #[inline(always)]
8681 fn kernel(
8682 token: Avx512,
8683 values: i8x32<Avx512>,
8684 mask: mask8x32<Avx512>,
8685 merge: i8x32<Avx512>,
8686 ) -> i8x32<Avx512> {
8687 _mm256_mask_compress_epi8(merge.into(), u64::from((mask).val) as u32, values.into())
8688 .simd_into(token)
8689 }
8690 );
8691 kernel(self, values, mask, merge)
8692 }
8693 #[inline(always)]
8694 fn expand_i8x32(self, values: i8x32<Self>, mask: mask8x32<Self>) -> i8x32<Self> {
8695 crate::kernel!(
8696 #[inline(always)]
8697 fn kernel(
8698 token: Avx512,
8699 values: i8x32<Avx512>,
8700 mask: mask8x32<Avx512>,
8701 ) -> i8x32<Avx512> {
8702 _mm256_maskz_expand_epi8(u64::from((mask).val) as u32, values.into())
8703 .simd_into(token)
8704 }
8705 );
8706 kernel(self, values, mask)
8707 }
8708 #[inline(always)]
8709 fn expand_merge_i8x32(
8710 self,
8711 values: i8x32<Self>,
8712 mask: mask8x32<Self>,
8713 merge: i8x32<Self>,
8714 ) -> i8x32<Self> {
8715 crate::kernel!(
8716 #[inline(always)]
8717 fn kernel(
8718 token: Avx512,
8719 values: i8x32<Avx512>,
8720 mask: mask8x32<Avx512>,
8721 merge: i8x32<Avx512>,
8722 ) -> i8x32<Avx512> {
8723 _mm256_mask_expand_epi8(merge.into(), u64::from((mask).val) as u32, values.into())
8724 .simd_into(token)
8725 }
8726 );
8727 kernel(self, values, mask, merge)
8728 }
8729 #[inline(always)]
8730 fn count_ones_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
8731 crate::kernel!(
8732 #[inline(always)]
8733 fn kernel(token: Avx512, a: i8x32<Avx512>) -> i8x32<Avx512> {
8734 _mm256_popcnt_epi8(a.into()).simd_into(token)
8735 }
8736 );
8737 kernel(self, a)
8738 }
8739 #[inline(always)]
8740 fn count_zeros_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
8741 self.count_ones_i8x32(self.not_i8x32(a))
8742 }
8743 #[inline(always)]
8744 fn add_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8745 crate::kernel!(
8746 #[inline(always)]
8747 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
8748 _mm256_add_epi8(a.into(), b.into()).simd_into(token)
8749 }
8750 );
8751 kernel(self, a, b)
8752 }
8753 #[inline(always)]
8754 fn saturating_add_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8755 crate::kernel!(
8756 #[inline(always)]
8757 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
8758 _mm256_adds_epi8(a.into(), b.into()).simd_into(token)
8759 }
8760 );
8761 kernel(self, a, b)
8762 }
8763 #[inline(always)]
8764 fn sub_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8765 crate::kernel!(
8766 #[inline(always)]
8767 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
8768 _mm256_sub_epi8(a.into(), b.into()).simd_into(token)
8769 }
8770 );
8771 kernel(self, a, b)
8772 }
8773 #[inline(always)]
8774 fn saturating_sub_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8775 crate::kernel!(
8776 #[inline(always)]
8777 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
8778 _mm256_subs_epi8(a.into(), b.into()).simd_into(token)
8779 }
8780 );
8781 kernel(self, a, b)
8782 }
8783 #[inline(always)]
8784 fn mul_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8785 crate::kernel!(
8786 #[inline(always)]
8787 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
8788 let a = a.into();
8789 let b = b.into();
8790 let low_mask = _mm256_set1_epi16(0xFF);
8791 let dst_even = _mm256_mullo_epi16(a, b);
8792 let dst_odd = _mm256_maddubs_epi16(a, _mm256_andnot_si256(low_mask, b));
8793 _mm256_or_si256(
8794 _mm256_slli_epi16(dst_odd, 8),
8795 _mm256_and_si256(dst_even, low_mask),
8796 )
8797 .simd_into(token)
8798 }
8799 );
8800 kernel(self, a, b)
8801 }
8802 #[inline(always)]
8803 fn and_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8804 crate::kernel!(
8805 #[inline(always)]
8806 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
8807 _mm256_and_si256(a.into(), b.into()).simd_into(token)
8808 }
8809 );
8810 kernel(self, a, b)
8811 }
8812 #[inline(always)]
8813 fn or_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8814 crate::kernel!(
8815 #[inline(always)]
8816 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
8817 _mm256_or_si256(a.into(), b.into()).simd_into(token)
8818 }
8819 );
8820 kernel(self, a, b)
8821 }
8822 #[inline(always)]
8823 fn xor_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8824 crate::kernel!(
8825 #[inline(always)]
8826 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
8827 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
8828 }
8829 );
8830 kernel(self, a, b)
8831 }
8832 #[inline(always)]
8833 fn not_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
8834 a ^ !0
8835 }
8836 #[inline(always)]
8837 fn shl_i8x32(self, a: i8x32<Self>, shift: u32) -> i8x32<Self> {
8838 crate::kernel!(
8839 #[inline(always)]
8840 fn kernel(token: Avx512, a: i8x32<Avx512>, shift: u32) -> i8x32<Avx512> {
8841 let val = a.into();
8842 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
8843 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
8844 let byte_mask = _mm256_set1_epi8(mask_byte);
8845 _mm256_sll_epi16(_mm256_and_si256(val, byte_mask), shift_count).simd_into(token)
8846 }
8847 );
8848 kernel(self, a, shift)
8849 }
8850 #[inline(always)]
8851 fn shlv_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8852 crate::kernel!(
8853 #[inline(always)]
8854 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
8855 let val = a.into();
8856 let counts = b.into();
8857 let byte_mask = _mm256_set1_epi16(0x00ff);
8858 let lo_counts = _mm256_and_si256(counts, byte_mask);
8859 let hi_counts = _mm256_srli_epi16::<8>(counts);
8860 let lo_shifted = _mm256_sllv_epi16(val, lo_counts);
8861 let hi_values = _mm256_andnot_si256(byte_mask, val);
8862 let hi_shifted = _mm256_sllv_epi16(hi_values, hi_counts);
8863 _mm256_mask_blend_epi8(0xaaaa_aaaa_u32, lo_shifted, hi_shifted).simd_into(token)
8864 }
8865 );
8866 kernel(self, a, b)
8867 }
8868 #[inline(always)]
8869 fn shr_i8x32(self, a: i8x32<Self>, shift: u32) -> i8x32<Self> {
8870 crate::kernel!(
8871 #[inline(always)]
8872 fn kernel(token: Avx512, a: i8x32<Avx512>, shift: u32) -> i8x32<Avx512> {
8873 let val = a.into();
8874 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
8875 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
8876 let byte_mask = _mm256_set1_epi8(mask_byte);
8877 let shifted = _mm256_srl_epi16(val, shift_count);
8878 let result = {
8879 let sign = _mm256_cmpgt_epi8(_mm256_setzero_si256(), val);
8880 _mm256_ternarylogic_epi32::<0xca>(byte_mask, shifted, sign)
8881 };
8882 result.simd_into(token)
8883 }
8884 );
8885 kernel(self, a, shift)
8886 }
8887 #[inline(always)]
8888 fn shrv_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8889 crate::kernel!(
8890 #[inline(always)]
8891 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
8892 let val = a.into();
8893 let counts = b.into();
8894 let zero = _mm256_setzero_si256();
8895 let value_extend = _mm256_cmpgt_epi8(zero, val);
8896 let lo_values = _mm256_unpacklo_epi8(val, value_extend);
8897 let hi_values = _mm256_unpackhi_epi8(val, value_extend);
8898 let lo_counts = _mm256_unpacklo_epi8(counts, zero);
8899 let hi_counts = _mm256_unpackhi_epi8(counts, zero);
8900 let byte_mask = _mm256_set1_epi16(0x00ff);
8901 let lo_shifted =
8902 _mm256_and_si256(_mm256_srav_epi16(lo_values, lo_counts), byte_mask);
8903 let hi_shifted =
8904 _mm256_and_si256(_mm256_srav_epi16(hi_values, hi_counts), byte_mask);
8905 _mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
8906 }
8907 );
8908 kernel(self, a, b)
8909 }
8910 #[inline(always)]
8911 fn max_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8912 crate::kernel!(
8913 #[inline(always)]
8914 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
8915 _mm256_max_epi8(a.into(), b.into()).simd_into(token)
8916 }
8917 );
8918 kernel(self, a, b)
8919 }
8920 #[inline(always)]
8921 fn min_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8922 crate::kernel!(
8923 #[inline(always)]
8924 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
8925 _mm256_min_epi8(a.into(), b.into()).simd_into(token)
8926 }
8927 );
8928 kernel(self, a, b)
8929 }
8930 #[inline(always)]
8931 fn clamp_i8x32(self, a: i8x32<Self>, b: i8x32<Self>, c: i8x32<Self>) -> i8x32<Self> {
8932 debug_assert!(
8933 b.simd_le(c).all_true(),
8934 "clamp requires min <= max for all elements"
8935 );
8936 a.max(b).min(c)
8937 }
8938 #[inline(always)]
8939 fn simd_eq_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
8940 crate::kernel!(
8941 #[inline(always)]
8942 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> mask8x32<Avx512> {
8943 mask8x32 {
8944 val: _mm256_cmpeq_epi8_mask(a.into(), b.into()),
8945 simd: token,
8946 }
8947 }
8948 );
8949 kernel(self, a, b)
8950 }
8951 #[inline(always)]
8952 fn simd_ne_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
8953 crate::kernel!(
8954 #[inline(always)]
8955 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> mask8x32<Avx512> {
8956 mask8x32 {
8957 val: _mm256_cmpneq_epi8_mask(a.into(), b.into()),
8958 simd: token,
8959 }
8960 }
8961 );
8962 kernel(self, a, b)
8963 }
8964 #[inline(always)]
8965 fn simd_lt_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
8966 crate::kernel!(
8967 #[inline(always)]
8968 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> mask8x32<Avx512> {
8969 mask8x32 {
8970 val: _mm256_cmplt_epi8_mask(a.into(), b.into()),
8971 simd: token,
8972 }
8973 }
8974 );
8975 kernel(self, a, b)
8976 }
8977 #[inline(always)]
8978 fn simd_le_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
8979 crate::kernel!(
8980 #[inline(always)]
8981 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> mask8x32<Avx512> {
8982 mask8x32 {
8983 val: _mm256_cmple_epi8_mask(a.into(), b.into()),
8984 simd: token,
8985 }
8986 }
8987 );
8988 kernel(self, a, b)
8989 }
8990 #[inline(always)]
8991 fn zip_low_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8992 crate::kernel!(
8993 #[inline(always)]
8994 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
8995 _mm256_permutex2var_epi8(
8996 a.into(),
8997 _mm256_setr_epi8(
8998 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41, 10,
8999 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47,
9000 ),
9001 b.into(),
9002 )
9003 .simd_into(token)
9004 }
9005 );
9006 kernel(self, a, b)
9007 }
9008 #[inline(always)]
9009 fn zip_high_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
9010 crate::kernel!(
9011 #[inline(always)]
9012 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
9013 _mm256_permutex2var_epi8(
9014 a.into(),
9015 _mm256_setr_epi8(
9016 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56, 25,
9017 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63,
9018 ),
9019 b.into(),
9020 )
9021 .simd_into(token)
9022 }
9023 );
9024 kernel(self, a, b)
9025 }
9026 #[inline(always)]
9027 fn unzip_low_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
9028 crate::kernel!(
9029 #[inline(always)]
9030 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
9031 _mm256_permutex2var_epi8(
9032 a.into(),
9033 _mm256_setr_epi8(
9034 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38,
9035 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62,
9036 ),
9037 b.into(),
9038 )
9039 .simd_into(token)
9040 }
9041 );
9042 kernel(self, a, b)
9043 }
9044 #[inline(always)]
9045 fn unzip_high_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
9046 crate::kernel!(
9047 #[inline(always)]
9048 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
9049 _mm256_permutex2var_epi8(
9050 a.into(),
9051 _mm256_setr_epi8(
9052 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39,
9053 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63,
9054 ),
9055 b.into(),
9056 )
9057 .simd_into(token)
9058 }
9059 );
9060 kernel(self, a, b)
9061 }
9062 #[inline(always)]
9063 fn interleave_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i8x32<Self>, i8x32<Self>) {
9064 crate::kernel!(
9065 #[inline(always)]
9066 fn kernel(
9067 token: Avx512,
9068 a: i8x32<Avx512>,
9069 b: i8x32<Avx512>,
9070 ) -> (i8x32<Avx512>, i8x32<Avx512>) {
9071 let a = a.into();
9072 let b = b.into();
9073 (
9074 _mm256_permutex2var_epi8(
9075 a,
9076 _mm256_setr_epi8(
9077 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41,
9078 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47,
9079 ),
9080 b,
9081 )
9082 .simd_into(token),
9083 _mm256_permutex2var_epi8(
9084 a,
9085 _mm256_setr_epi8(
9086 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56,
9087 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63,
9088 ),
9089 b,
9090 )
9091 .simd_into(token),
9092 )
9093 }
9094 );
9095 kernel(self, a, b)
9096 }
9097 #[inline(always)]
9098 fn deinterleave_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i8x32<Self>, i8x32<Self>) {
9099 crate::kernel!(
9100 #[inline(always)]
9101 fn kernel(
9102 token: Avx512,
9103 a: i8x32<Avx512>,
9104 b: i8x32<Avx512>,
9105 ) -> (i8x32<Avx512>, i8x32<Avx512>) {
9106 let a = a.into();
9107 let b = b.into();
9108 (
9109 _mm256_permutex2var_epi8(
9110 a,
9111 _mm256_setr_epi8(
9112 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36,
9113 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62,
9114 ),
9115 b,
9116 )
9117 .simd_into(token),
9118 _mm256_permutex2var_epi8(
9119 a,
9120 _mm256_setr_epi8(
9121 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37,
9122 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63,
9123 ),
9124 b,
9125 )
9126 .simd_into(token),
9127 )
9128 }
9129 );
9130 kernel(self, a, b)
9131 }
9132 #[inline(always)]
9133 fn select_i8x32(self, a: mask8x32<Self>, b: i8x32<Self>, c: i8x32<Self>) -> i8x32<Self> {
9134 crate::kernel!(
9135 #[inline(always)]
9136 fn kernel(
9137 token: Avx512,
9138 a: mask8x32<Avx512>,
9139 b: i8x32<Avx512>,
9140 c: i8x32<Avx512>,
9141 ) -> i8x32<Avx512> {
9142 _mm256_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token)
9143 }
9144 );
9145 kernel(self, a, b, c)
9146 }
9147 #[inline(always)]
9148 fn combine_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x64<Self> {
9149 crate::kernel!(
9150 #[inline(always)]
9151 fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x64<Avx512> {
9152 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
9153 }
9154 );
9155 kernel(self, a, b)
9156 }
9157 #[inline(always)]
9158 fn split_i8x32(self, a: i8x32<Self>) -> (i8x16<Self>, i8x16<Self>) {
9159 crate::kernel!(
9160 #[inline(always)]
9161 fn kernel(token: Avx512, a: i8x32<Avx512>) -> (i8x16<Avx512>, i8x16<Avx512>) {
9162 (
9163 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
9164 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
9165 )
9166 }
9167 );
9168 kernel(self, a)
9169 }
9170 #[inline(always)]
9171 fn neg_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
9172 crate::kernel!(
9173 #[inline(always)]
9174 fn kernel(token: Avx512, a: i8x32<Avx512>) -> i8x32<Avx512> {
9175 _mm256_sub_epi8(_mm256_setzero_si256(), a.into()).simd_into(token)
9176 }
9177 );
9178 kernel(self, a)
9179 }
9180 #[inline(always)]
9181 fn widen_i8x32(self, a: i8x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
9182 crate::kernel!(
9183 #[inline(always)]
9184 fn kernel(token: Avx512, a: i8x32<Avx512>) -> (i16x16<Avx512>, i16x16<Avx512>) {
9185 let raw = a.into();
9186 (
9187 _mm256_cvtepi8_epi16(_mm256_castsi256_si128(raw)).simd_into(token),
9188 _mm256_cvtepi8_epi16(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
9189 )
9190 }
9191 );
9192 kernel(self, a)
9193 }
9194 #[inline(always)]
9195 fn widening_mul_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
9196 let (a0, a1) = self.widen_i8x32(a);
9197 let (b0, b1) = self.widen_i8x32(b);
9198 (self.mul_i16x16(a0, b0), self.mul_i16x16(a1, b1))
9199 }
9200 #[inline(always)]
9201 fn widening_add_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
9202 crate::kernel!(
9203 #[inline(always)]
9204 fn kernel(
9205 token: Avx512,
9206 a: i8x32<Avx512>,
9207 b: i8x32<Avx512>,
9208 ) -> (i16x16<Avx512>, i16x16<Avx512>) {
9209 let a = a.into();
9210 let b = b.into();
9211 let low = a;
9212 let high = b;
9213 let result_low = _mm256_unpacklo_epi8(low, high);
9214 let result_high = _mm256_unpackhi_epi8(low, high);
9215 let ones = _mm256_set1_epi8(1);
9216 let result_low = _mm256_maddubs_epi16(ones, result_low);
9217 let result_high = _mm256_maddubs_epi16(ones, result_high);
9218 let lo = _mm256_permute2x128_si256::<0x20>(result_low, result_high);
9219 let hi = _mm256_permute2x128_si256::<0x31>(result_low, result_high);
9220 let result_low = lo;
9221 let result_high = hi;
9222 (result_low.simd_into(token), result_high.simd_into(token))
9223 }
9224 );
9225 kernel(self, a, b)
9226 }
9227 #[inline(always)]
9228 fn pairwise_widening_add_i8x32(self, a: i8x32<Self>) -> i16x16<Self> {
9229 crate::kernel!(
9230 #[inline(always)]
9231 fn kernel(token: Avx512, a: i8x32<Avx512>) -> i16x16<Avx512> {
9232 ({ _mm256_maddubs_epi16(_mm256_set1_epi8(1), a.into()) }).simd_into(token)
9233 }
9234 );
9235 kernel(self, a)
9236 }
9237 #[inline(always)]
9238 fn pairwise_dot_product_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i16x16<Self> {
9239 let (low, high) = self.widening_mul_i8x32(a, b);
9240 let (even, odd) = self.deinterleave_i16x16(low, high);
9241 self.add_i16x16(even, odd)
9242 }
9243 #[inline(always)]
9244 fn splat_u8x32(self, val: u8) -> u8x32<Self> {
9245 crate::kernel!(
9246 #[inline(always)]
9247 fn kernel(token: Avx512, val: u8) -> u8x32<Avx512> {
9248 _mm256_set1_epi8(val.cast_signed()).simd_into(token)
9249 }
9250 );
9251 kernel(self, val)
9252 }
9253 #[inline(always)]
9254 fn reverse_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
9255 let indices: u8x32<Self> = [
9256 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10,
9257 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
9258 ]
9259 .simd_into(self);
9260 self.swizzle_dyn_u8x32(a, indices)
9261 }
9262 #[inline(always)]
9263 fn slide_u8x32<const SHIFT: usize>(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9264 crate::kernel!(
9265 #[inline(always)]
9266 fn kernel(
9267 token: Avx512,
9268 a: u8x32<Avx512>,
9269 b: u8x32<Avx512>,
9270 shift: usize,
9271 ) -> u8x32<Avx512> {
9272 if shift >= 32usize {
9273 return b;
9274 }
9275 let idx = _mm256_add_epi8(
9276 _mm256_setr_epi8(
9277 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
9278 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
9279 ),
9280 _mm256_set1_epi8((shift) as i8),
9281 );
9282 let result = _mm256_permutex2var_epi8(
9283 Bytes::to_bytes(a).val.0,
9284 idx,
9285 Bytes::to_bytes(b).val.0,
9286 );
9287 Bytes::from_bytes(u8x32 {
9288 val: crate::support::Aligned256(result),
9289 simd: token,
9290 })
9291 }
9292 );
9293 kernel(self, a, b, SHIFT)
9294 }
9295 #[inline(always)]
9296 fn slide_within_blocks_u8x32<const SHIFT: usize>(
9297 self,
9298 a: u8x32<Self>,
9299 b: u8x32<Self>,
9300 ) -> u8x32<Self> {
9301 if SHIFT == 0 {
9302 return a;
9303 }
9304 if SHIFT >= 16usize {
9305 return b;
9306 }
9307 let a = Bytes::to_bytes(a).val.0;
9308 let b = Bytes::to_bytes(b).val.0;
9309 let result = dyn_alignr_256(self, b, a, SHIFT);
9310 Bytes::from_bytes(u8x32 {
9311 val: crate::support::Aligned256(result),
9312 simd: self,
9313 })
9314 }
9315 #[inline(always)]
9316 fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
9317 crate::kernel!(
9318 #[inline(always)]
9319 fn kernel(token: Avx512, a: u8x32<Avx512>, indices: u8x32<Avx512>) -> u8x32<Avx512> {
9320 let bytes = Bytes::to_bytes(a).val.0;
9321 let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into());
9322 Bytes::from_bytes(u8x32 {
9323 val: crate::support::Aligned256(result),
9324 simd: token,
9325 })
9326 }
9327 );
9328 kernel(self, a, indices)
9329 }
9330 #[inline(always)]
9331 fn swizzle_dyn_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
9332 crate::kernel!(
9333 #[inline(always)]
9334 fn kernel(token: Avx512, a: u8x32<Avx512>, indices: u8x32<Avx512>) -> u8x32<Avx512> {
9335 let result = _mm256_permutexvar_epi8(indices.into(), Bytes::to_bytes(a).val.0);
9336 Bytes::from_bytes(u8x32 {
9337 val: crate::support::Aligned256(result),
9338 simd: token,
9339 })
9340 }
9341 );
9342 kernel(self, a, indices)
9343 }
9344 #[inline(always)]
9345 fn swizzle_dyn_precise_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
9346 crate::kernel!(
9347 #[inline(always)]
9348 fn kernel(token: Avx512, a: u8x32<Avx512>, indices: u8x32<Avx512>) -> u8x32<Avx512> {
9349 let bytes = Bytes::to_bytes(a).val.0;
9350 let indices = indices.into();
9351 let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32));
9352 let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256());
9353 let result_bytes = u8x32 {
9354 val: crate::support::Aligned256(result),
9355 simd: token,
9356 };
9357 Bytes::from_bytes(result_bytes)
9358 }
9359 );
9360 kernel(self, a, indices)
9361 }
9362 #[inline(always)]
9363 fn concat_swizzle_dyn_u8x32(
9364 self,
9365 a: u8x32<Self>,
9366 b: u8x32<Self>,
9367 indices: u8x32<Self>,
9368 ) -> u8x32<Self> {
9369 crate::kernel!(
9370 #[inline(always)]
9371 fn kernel(
9372 token: Avx512,
9373 a: u8x32<Avx512>,
9374 b: u8x32<Avx512>,
9375 indices: u8x32<Avx512>,
9376 ) -> u8x32<Avx512> {
9377 let result = _mm256_permutex2var_epi8(
9378 Bytes::to_bytes(a).val.0,
9379 indices.into(),
9380 Bytes::to_bytes(b).val.0,
9381 );
9382 Bytes::from_bytes(u8x32 {
9383 val: crate::support::Aligned256(result),
9384 simd: token,
9385 })
9386 }
9387 );
9388 kernel(self, a, b, indices)
9389 }
9390 #[inline(always)]
9391 fn concat_swizzle_dyn_precise_u8x32(
9392 self,
9393 a: u8x32<Self>,
9394 b: u8x32<Self>,
9395 indices: u8x32<Self>,
9396 ) -> u8x32<Self> {
9397 crate::kernel!(
9398 #[inline(always)]
9399 fn kernel(
9400 token: Avx512,
9401 a: u8x32<Avx512>,
9402 b: u8x32<Avx512>,
9403 indices: u8x32<Avx512>,
9404 ) -> u8x32<Avx512> {
9405 let a_bytes = Bytes::to_bytes(a).val.0;
9406 let b_bytes = Bytes::to_bytes(b).val.0;
9407 let indices = indices.into();
9408 let in_range = _mm256_cmplt_epu8_mask(indices, _mm256_set1_epi8(64));
9409 let result = _mm256_maskz_permutex2var_epi8(in_range, a_bytes, indices, b_bytes);
9410 let result_bytes = u8x32 {
9411 val: crate::support::Aligned256(result),
9412 simd: token,
9413 };
9414 Bytes::from_bytes(result_bytes)
9415 }
9416 );
9417 kernel(self, a, b, indices)
9418 }
9419 #[inline(always)]
9420 fn compress_u8x32(self, values: u8x32<Self>, mask: mask8x32<Self>) -> u8x32<Self> {
9421 crate::kernel!(
9422 #[inline(always)]
9423 fn kernel(
9424 token: Avx512,
9425 values: u8x32<Avx512>,
9426 mask: mask8x32<Avx512>,
9427 ) -> u8x32<Avx512> {
9428 _mm256_maskz_compress_epi8(u64::from((mask).val) as u32, values.into())
9429 .simd_into(token)
9430 }
9431 );
9432 kernel(self, values, mask)
9433 }
9434 #[inline(always)]
9435 fn compress_merge_u8x32(
9436 self,
9437 values: u8x32<Self>,
9438 mask: mask8x32<Self>,
9439 merge: u8x32<Self>,
9440 ) -> u8x32<Self> {
9441 crate::kernel!(
9442 #[inline(always)]
9443 fn kernel(
9444 token: Avx512,
9445 values: u8x32<Avx512>,
9446 mask: mask8x32<Avx512>,
9447 merge: u8x32<Avx512>,
9448 ) -> u8x32<Avx512> {
9449 _mm256_mask_compress_epi8(merge.into(), u64::from((mask).val) as u32, values.into())
9450 .simd_into(token)
9451 }
9452 );
9453 kernel(self, values, mask, merge)
9454 }
9455 #[inline(always)]
9456 fn expand_u8x32(self, values: u8x32<Self>, mask: mask8x32<Self>) -> u8x32<Self> {
9457 crate::kernel!(
9458 #[inline(always)]
9459 fn kernel(
9460 token: Avx512,
9461 values: u8x32<Avx512>,
9462 mask: mask8x32<Avx512>,
9463 ) -> u8x32<Avx512> {
9464 _mm256_maskz_expand_epi8(u64::from((mask).val) as u32, values.into())
9465 .simd_into(token)
9466 }
9467 );
9468 kernel(self, values, mask)
9469 }
9470 #[inline(always)]
9471 fn expand_merge_u8x32(
9472 self,
9473 values: u8x32<Self>,
9474 mask: mask8x32<Self>,
9475 merge: u8x32<Self>,
9476 ) -> u8x32<Self> {
9477 crate::kernel!(
9478 #[inline(always)]
9479 fn kernel(
9480 token: Avx512,
9481 values: u8x32<Avx512>,
9482 mask: mask8x32<Avx512>,
9483 merge: u8x32<Avx512>,
9484 ) -> u8x32<Avx512> {
9485 _mm256_mask_expand_epi8(merge.into(), u64::from((mask).val) as u32, values.into())
9486 .simd_into(token)
9487 }
9488 );
9489 kernel(self, values, mask, merge)
9490 }
9491 #[inline(always)]
9492 fn count_ones_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
9493 crate::kernel!(
9494 #[inline(always)]
9495 fn kernel(token: Avx512, a: u8x32<Avx512>) -> u8x32<Avx512> {
9496 _mm256_popcnt_epi8(a.into()).simd_into(token)
9497 }
9498 );
9499 kernel(self, a)
9500 }
9501 #[inline(always)]
9502 fn count_zeros_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
9503 self.count_ones_u8x32(self.not_u8x32(a))
9504 }
9505 #[inline(always)]
9506 fn add_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9507 crate::kernel!(
9508 #[inline(always)]
9509 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9510 _mm256_add_epi8(a.into(), b.into()).simd_into(token)
9511 }
9512 );
9513 kernel(self, a, b)
9514 }
9515 #[inline(always)]
9516 fn saturating_add_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9517 crate::kernel!(
9518 #[inline(always)]
9519 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9520 _mm256_adds_epu8(a.into(), b.into()).simd_into(token)
9521 }
9522 );
9523 kernel(self, a, b)
9524 }
9525 #[inline(always)]
9526 fn sub_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9527 crate::kernel!(
9528 #[inline(always)]
9529 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9530 _mm256_sub_epi8(a.into(), b.into()).simd_into(token)
9531 }
9532 );
9533 kernel(self, a, b)
9534 }
9535 #[inline(always)]
9536 fn saturating_sub_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9537 crate::kernel!(
9538 #[inline(always)]
9539 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9540 _mm256_subs_epu8(a.into(), b.into()).simd_into(token)
9541 }
9542 );
9543 kernel(self, a, b)
9544 }
9545 #[inline(always)]
9546 fn mul_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9547 crate::kernel!(
9548 #[inline(always)]
9549 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9550 let a = a.into();
9551 let b = b.into();
9552 let low_mask = _mm256_set1_epi16(0xFF);
9553 let dst_even = _mm256_mullo_epi16(a, b);
9554 let dst_odd = _mm256_maddubs_epi16(a, _mm256_andnot_si256(low_mask, b));
9555 _mm256_or_si256(
9556 _mm256_slli_epi16(dst_odd, 8),
9557 _mm256_and_si256(dst_even, low_mask),
9558 )
9559 .simd_into(token)
9560 }
9561 );
9562 kernel(self, a, b)
9563 }
9564 #[inline(always)]
9565 fn and_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9566 crate::kernel!(
9567 #[inline(always)]
9568 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9569 _mm256_and_si256(a.into(), b.into()).simd_into(token)
9570 }
9571 );
9572 kernel(self, a, b)
9573 }
9574 #[inline(always)]
9575 fn or_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9576 crate::kernel!(
9577 #[inline(always)]
9578 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9579 _mm256_or_si256(a.into(), b.into()).simd_into(token)
9580 }
9581 );
9582 kernel(self, a, b)
9583 }
9584 #[inline(always)]
9585 fn xor_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9586 crate::kernel!(
9587 #[inline(always)]
9588 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9589 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
9590 }
9591 );
9592 kernel(self, a, b)
9593 }
9594 #[inline(always)]
9595 fn not_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
9596 a ^ !0
9597 }
9598 #[inline(always)]
9599 fn shl_u8x32(self, a: u8x32<Self>, shift: u32) -> u8x32<Self> {
9600 crate::kernel!(
9601 #[inline(always)]
9602 fn kernel(token: Avx512, a: u8x32<Avx512>, shift: u32) -> u8x32<Avx512> {
9603 let val = a.into();
9604 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
9605 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
9606 let byte_mask = _mm256_set1_epi8(mask_byte);
9607 _mm256_sll_epi16(_mm256_and_si256(val, byte_mask), shift_count).simd_into(token)
9608 }
9609 );
9610 kernel(self, a, shift)
9611 }
9612 #[inline(always)]
9613 fn shlv_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9614 crate::kernel!(
9615 #[inline(always)]
9616 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9617 let val = a.into();
9618 let counts = b.into();
9619 let byte_mask = _mm256_set1_epi16(0x00ff);
9620 let lo_counts = _mm256_and_si256(counts, byte_mask);
9621 let hi_counts = _mm256_srli_epi16::<8>(counts);
9622 let lo_shifted = _mm256_sllv_epi16(val, lo_counts);
9623 let hi_values = _mm256_andnot_si256(byte_mask, val);
9624 let hi_shifted = _mm256_sllv_epi16(hi_values, hi_counts);
9625 _mm256_mask_blend_epi8(0xaaaa_aaaa_u32, lo_shifted, hi_shifted).simd_into(token)
9626 }
9627 );
9628 kernel(self, a, b)
9629 }
9630 #[inline(always)]
9631 fn shr_u8x32(self, a: u8x32<Self>, shift: u32) -> u8x32<Self> {
9632 crate::kernel!(
9633 #[inline(always)]
9634 fn kernel(token: Avx512, a: u8x32<Avx512>, shift: u32) -> u8x32<Avx512> {
9635 let val = a.into();
9636 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
9637 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
9638 let byte_mask = _mm256_set1_epi8(mask_byte);
9639 let shifted = _mm256_srl_epi16(val, shift_count);
9640 let result = { _mm256_and_si256(shifted, byte_mask) };
9641 result.simd_into(token)
9642 }
9643 );
9644 kernel(self, a, shift)
9645 }
9646 #[inline(always)]
9647 fn shrv_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9648 crate::kernel!(
9649 #[inline(always)]
9650 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9651 let val = a.into();
9652 let counts = b.into();
9653 let zero = _mm256_setzero_si256();
9654 let value_extend = zero;
9655 let lo_values = _mm256_unpacklo_epi8(val, value_extend);
9656 let hi_values = _mm256_unpackhi_epi8(val, value_extend);
9657 let lo_counts = _mm256_unpacklo_epi8(counts, zero);
9658 let hi_counts = _mm256_unpackhi_epi8(counts, zero);
9659 let byte_mask = _mm256_set1_epi16(0x00ff);
9660 let lo_shifted =
9661 _mm256_and_si256(_mm256_srlv_epi16(lo_values, lo_counts), byte_mask);
9662 let hi_shifted =
9663 _mm256_and_si256(_mm256_srlv_epi16(hi_values, hi_counts), byte_mask);
9664 _mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
9665 }
9666 );
9667 kernel(self, a, b)
9668 }
9669 #[inline(always)]
9670 fn max_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9671 crate::kernel!(
9672 #[inline(always)]
9673 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9674 _mm256_max_epu8(a.into(), b.into()).simd_into(token)
9675 }
9676 );
9677 kernel(self, a, b)
9678 }
9679 #[inline(always)]
9680 fn min_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9681 crate::kernel!(
9682 #[inline(always)]
9683 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9684 _mm256_min_epu8(a.into(), b.into()).simd_into(token)
9685 }
9686 );
9687 kernel(self, a, b)
9688 }
9689 #[inline(always)]
9690 fn clamp_u8x32(self, a: u8x32<Self>, b: u8x32<Self>, c: u8x32<Self>) -> u8x32<Self> {
9691 debug_assert!(
9692 b.simd_le(c).all_true(),
9693 "clamp requires min <= max for all elements"
9694 );
9695 a.max(b).min(c)
9696 }
9697 #[inline(always)]
9698 fn simd_eq_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
9699 crate::kernel!(
9700 #[inline(always)]
9701 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> mask8x32<Avx512> {
9702 mask8x32 {
9703 val: _mm256_cmpeq_epu8_mask(a.into(), b.into()),
9704 simd: token,
9705 }
9706 }
9707 );
9708 kernel(self, a, b)
9709 }
9710 #[inline(always)]
9711 fn simd_ne_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
9712 crate::kernel!(
9713 #[inline(always)]
9714 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> mask8x32<Avx512> {
9715 mask8x32 {
9716 val: _mm256_cmpneq_epu8_mask(a.into(), b.into()),
9717 simd: token,
9718 }
9719 }
9720 );
9721 kernel(self, a, b)
9722 }
9723 #[inline(always)]
9724 fn simd_lt_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
9725 crate::kernel!(
9726 #[inline(always)]
9727 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> mask8x32<Avx512> {
9728 mask8x32 {
9729 val: _mm256_cmplt_epu8_mask(a.into(), b.into()),
9730 simd: token,
9731 }
9732 }
9733 );
9734 kernel(self, a, b)
9735 }
9736 #[inline(always)]
9737 fn simd_le_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
9738 crate::kernel!(
9739 #[inline(always)]
9740 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> mask8x32<Avx512> {
9741 mask8x32 {
9742 val: _mm256_cmple_epu8_mask(a.into(), b.into()),
9743 simd: token,
9744 }
9745 }
9746 );
9747 kernel(self, a, b)
9748 }
9749 #[inline(always)]
9750 fn zip_low_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9751 crate::kernel!(
9752 #[inline(always)]
9753 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9754 _mm256_permutex2var_epi8(
9755 a.into(),
9756 _mm256_setr_epi8(
9757 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41, 10,
9758 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47,
9759 ),
9760 b.into(),
9761 )
9762 .simd_into(token)
9763 }
9764 );
9765 kernel(self, a, b)
9766 }
9767 #[inline(always)]
9768 fn zip_high_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9769 crate::kernel!(
9770 #[inline(always)]
9771 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9772 _mm256_permutex2var_epi8(
9773 a.into(),
9774 _mm256_setr_epi8(
9775 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56, 25,
9776 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63,
9777 ),
9778 b.into(),
9779 )
9780 .simd_into(token)
9781 }
9782 );
9783 kernel(self, a, b)
9784 }
9785 #[inline(always)]
9786 fn unzip_low_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9787 crate::kernel!(
9788 #[inline(always)]
9789 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9790 _mm256_permutex2var_epi8(
9791 a.into(),
9792 _mm256_setr_epi8(
9793 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38,
9794 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62,
9795 ),
9796 b.into(),
9797 )
9798 .simd_into(token)
9799 }
9800 );
9801 kernel(self, a, b)
9802 }
9803 #[inline(always)]
9804 fn unzip_high_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9805 crate::kernel!(
9806 #[inline(always)]
9807 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
9808 _mm256_permutex2var_epi8(
9809 a.into(),
9810 _mm256_setr_epi8(
9811 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39,
9812 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63,
9813 ),
9814 b.into(),
9815 )
9816 .simd_into(token)
9817 }
9818 );
9819 kernel(self, a, b)
9820 }
9821 #[inline(always)]
9822 fn interleave_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u8x32<Self>, u8x32<Self>) {
9823 crate::kernel!(
9824 #[inline(always)]
9825 fn kernel(
9826 token: Avx512,
9827 a: u8x32<Avx512>,
9828 b: u8x32<Avx512>,
9829 ) -> (u8x32<Avx512>, u8x32<Avx512>) {
9830 let a = a.into();
9831 let b = b.into();
9832 (
9833 _mm256_permutex2var_epi8(
9834 a,
9835 _mm256_setr_epi8(
9836 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41,
9837 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47,
9838 ),
9839 b,
9840 )
9841 .simd_into(token),
9842 _mm256_permutex2var_epi8(
9843 a,
9844 _mm256_setr_epi8(
9845 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56,
9846 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63,
9847 ),
9848 b,
9849 )
9850 .simd_into(token),
9851 )
9852 }
9853 );
9854 kernel(self, a, b)
9855 }
9856 #[inline(always)]
9857 fn deinterleave_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u8x32<Self>, u8x32<Self>) {
9858 crate::kernel!(
9859 #[inline(always)]
9860 fn kernel(
9861 token: Avx512,
9862 a: u8x32<Avx512>,
9863 b: u8x32<Avx512>,
9864 ) -> (u8x32<Avx512>, u8x32<Avx512>) {
9865 let a = a.into();
9866 let b = b.into();
9867 (
9868 _mm256_permutex2var_epi8(
9869 a,
9870 _mm256_setr_epi8(
9871 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36,
9872 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62,
9873 ),
9874 b,
9875 )
9876 .simd_into(token),
9877 _mm256_permutex2var_epi8(
9878 a,
9879 _mm256_setr_epi8(
9880 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37,
9881 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63,
9882 ),
9883 b,
9884 )
9885 .simd_into(token),
9886 )
9887 }
9888 );
9889 kernel(self, a, b)
9890 }
9891 #[inline(always)]
9892 fn select_u8x32(self, a: mask8x32<Self>, b: u8x32<Self>, c: u8x32<Self>) -> u8x32<Self> {
9893 crate::kernel!(
9894 #[inline(always)]
9895 fn kernel(
9896 token: Avx512,
9897 a: mask8x32<Avx512>,
9898 b: u8x32<Avx512>,
9899 c: u8x32<Avx512>,
9900 ) -> u8x32<Avx512> {
9901 _mm256_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token)
9902 }
9903 );
9904 kernel(self, a, b, c)
9905 }
9906 #[inline(always)]
9907 fn combine_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x64<Self> {
9908 crate::kernel!(
9909 #[inline(always)]
9910 fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x64<Avx512> {
9911 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
9912 }
9913 );
9914 kernel(self, a, b)
9915 }
9916 #[inline(always)]
9917 fn split_u8x32(self, a: u8x32<Self>) -> (u8x16<Self>, u8x16<Self>) {
9918 crate::kernel!(
9919 #[inline(always)]
9920 fn kernel(token: Avx512, a: u8x32<Avx512>) -> (u8x16<Avx512>, u8x16<Avx512>) {
9921 (
9922 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
9923 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
9924 )
9925 }
9926 );
9927 kernel(self, a)
9928 }
9929 #[inline(always)]
9930 fn widen_u8x32(self, a: u8x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
9931 crate::kernel!(
9932 #[inline(always)]
9933 fn kernel(token: Avx512, a: u8x32<Avx512>) -> (u16x16<Avx512>, u16x16<Avx512>) {
9934 let raw = a.into();
9935 (
9936 _mm256_cvtepu8_epi16(_mm256_castsi256_si128(raw)).simd_into(token),
9937 _mm256_cvtepu8_epi16(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
9938 )
9939 }
9940 );
9941 kernel(self, a)
9942 }
9943 #[inline(always)]
9944 fn widening_mul_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
9945 let (a0, a1) = self.widen_u8x32(a);
9946 let (b0, b1) = self.widen_u8x32(b);
9947 (self.mul_u16x16(a0, b0), self.mul_u16x16(a1, b1))
9948 }
9949 #[inline(always)]
9950 fn widening_add_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
9951 crate::kernel!(
9952 #[inline(always)]
9953 fn kernel(
9954 token: Avx512,
9955 a: u8x32<Avx512>,
9956 b: u8x32<Avx512>,
9957 ) -> (u16x16<Avx512>, u16x16<Avx512>) {
9958 let a = a.into();
9959 let b = b.into();
9960 let low = a;
9961 let high = b;
9962 let result_low = _mm256_unpacklo_epi8(low, high);
9963 let result_high = _mm256_unpackhi_epi8(low, high);
9964 let ones = _mm256_set1_epi8(1);
9965 let result_low = _mm256_maddubs_epi16(result_low, ones);
9966 let result_high = _mm256_maddubs_epi16(result_high, ones);
9967 let lo = _mm256_permute2x128_si256::<0x20>(result_low, result_high);
9968 let hi = _mm256_permute2x128_si256::<0x31>(result_low, result_high);
9969 let result_low = lo;
9970 let result_high = hi;
9971 (result_low.simd_into(token), result_high.simd_into(token))
9972 }
9973 );
9974 kernel(self, a, b)
9975 }
9976 #[inline(always)]
9977 fn pairwise_widening_add_u8x32(self, a: u8x32<Self>) -> u16x16<Self> {
9978 crate::kernel!(
9979 #[inline(always)]
9980 fn kernel(token: Avx512, a: u8x32<Avx512>) -> u16x16<Avx512> {
9981 ({ _mm256_maddubs_epi16(a.into(), _mm256_set1_epi8(1)) }).simd_into(token)
9982 }
9983 );
9984 kernel(self, a)
9985 }
9986 #[inline(always)]
9987 fn pairwise_dot_product_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u16x16<Self> {
9988 let (low, high) = self.widening_mul_u8x32(a, b);
9989 let (even, odd) = self.deinterleave_u16x16(low, high);
9990 self.add_u16x16(even, odd)
9991 }
9992 #[inline(always)]
9993 fn splat_mask8x32(self, val: bool) -> mask8x32<Self> {
9994 mask8x32 {
9995 val: (if val { 4294967295u64 } else { 0 }) as _,
9996 simd: self,
9997 }
9998 }
9999 #[inline(always)]
10000 fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32<Self> {
10001 mask8x32 {
10002 val: (bits & 4294967295u64) as _,
10003 simd: self,
10004 }
10005 }
10006 #[inline(always)]
10007 fn to_bitmask_mask8x32(self, a: mask8x32<Self>) -> u64 {
10008 u64::from((a).val) & 4294967295u64
10009 }
10010 #[inline(always)]
10011 fn from_vector_mask8x32(self, vector: i8x32<Self>) -> mask8x32<Self> {
10012 crate::kernel!(
10013 #[inline(always)]
10014 fn kernel(token: Avx512, vector: i8x32<Avx512>) -> mask8x32<Avx512> {
10015 _mm256_movepi8_mask(vector.into()).simd_into(token)
10016 }
10017 );
10018 kernel(self, vector)
10019 }
10020 #[inline(always)]
10021 fn to_vector_mask8x32(self, a: mask8x32<Self>) -> i8x32<Self> {
10022 crate::kernel!(
10023 #[inline(always)]
10024 fn kernel(token: Avx512, a: mask8x32<Avx512>) -> i8x32<Avx512> {
10025 _mm256_movm_epi8(a.val).simd_into(token)
10026 }
10027 );
10028 kernel(self, a)
10029 }
10030 #[inline(always)]
10031 fn set_mask8x32(self, a: &mut mask8x32<Self>, index: usize, value: bool) -> () {
10032 assert!(
10033 index < 32usize,
10034 "mask lane index {index} is out of bounds for {} lanes",
10035 32usize
10036 );
10037 let bit = 1u64 << index;
10038 let bits = u64::from((a).val);
10039 let bits = if value { bits | bit } else { bits & !bit };
10040 *a = mask8x32 {
10041 val: (bits) as _,
10042 simd: self,
10043 };
10044 }
10045 #[inline(always)]
10046 fn rotate_elements_left_mask8x32<const OFFSET: usize>(
10047 self,
10048 a: mask8x32<Self>,
10049 ) -> mask8x32<Self> {
10050 mask8x32 {
10051 val: (a.val.rotate_right((OFFSET % 32) as u32)) as _,
10052 simd: self,
10053 }
10054 }
10055 #[inline(always)]
10056 fn rotate_elements_right_mask8x32<const OFFSET: usize>(
10057 self,
10058 a: mask8x32<Self>,
10059 ) -> mask8x32<Self> {
10060 mask8x32 {
10061 val: (a.val.rotate_left((OFFSET % 32) as u32)) as _,
10062 simd: self,
10063 }
10064 }
10065 #[inline(always)]
10066 fn and_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
10067 mask8x32 {
10068 val: ((u64::from((a).val) & u64::from((b).val)) & 4294967295u64) as _,
10069 simd: self,
10070 }
10071 }
10072 #[inline(always)]
10073 fn or_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
10074 mask8x32 {
10075 val: ((u64::from((a).val) | u64::from((b).val)) & 4294967295u64) as _,
10076 simd: self,
10077 }
10078 }
10079 #[inline(always)]
10080 fn xor_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
10081 mask8x32 {
10082 val: ((u64::from((a).val) ^ u64::from((b).val)) & 4294967295u64) as _,
10083 simd: self,
10084 }
10085 }
10086 #[inline(always)]
10087 fn not_mask8x32(self, a: mask8x32<Self>) -> mask8x32<Self> {
10088 mask8x32 {
10089 val: ((!u64::from((a).val)) & 4294967295u64) as _,
10090 simd: self,
10091 }
10092 }
10093 #[inline(always)]
10094 fn reverse_mask8x32(self, a: mask8x32<Self>) -> mask8x32<Self> {
10095 mask8x32 {
10096 val: (a.val.reverse_bits() >> 0usize) as _,
10097 simd: self,
10098 }
10099 }
10100 #[inline(always)]
10101 fn select_mask8x32(
10102 self,
10103 a: mask8x32<Self>,
10104 b: mask8x32<Self>,
10105 c: mask8x32<Self>,
10106 ) -> mask8x32<Self> {
10107 mask8x32 {
10108 val: (((u64::from((a).val) & u64::from((b).val))
10109 | ((!u64::from((a).val)) & u64::from((c).val)))
10110 & 4294967295u64) as _,
10111 simd: self,
10112 }
10113 }
10114 #[inline(always)]
10115 fn simd_eq_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
10116 mask8x32 {
10117 val: (!u64::from(a.val ^ b.val) & 4294967295u64) as _,
10118 simd: self,
10119 }
10120 }
10121 #[inline(always)]
10122 fn simd_ne_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
10123 mask8x32 {
10124 val: (u64::from(a.val ^ b.val) & 4294967295u64) as _,
10125 simd: self,
10126 }
10127 }
10128 #[inline(always)]
10129 fn any_true_mask8x32(self, a: mask8x32<Self>) -> bool {
10130 let bits = u64::from((a).val) & 4294967295u64;
10131 bits != 0
10132 }
10133 #[inline(always)]
10134 fn all_true_mask8x32(self, a: mask8x32<Self>) -> bool {
10135 let bits = u64::from((a).val) & 4294967295u64;
10136 bits == 4294967295u64
10137 }
10138 #[inline(always)]
10139 fn any_false_mask8x32(self, a: mask8x32<Self>) -> bool {
10140 let bits = u64::from((a).val) & 4294967295u64;
10141 bits != 4294967295u64
10142 }
10143 #[inline(always)]
10144 fn all_false_mask8x32(self, a: mask8x32<Self>) -> bool {
10145 let bits = u64::from((a).val) & 4294967295u64;
10146 bits == 0
10147 }
10148 #[inline(always)]
10149 fn combine_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x64<Self> {
10150 let bits = (u64::from(a.val) | (u64::from(b.val) << 32usize)) & u64::MAX;
10151 mask8x64 {
10152 val: bits,
10153 simd: self,
10154 }
10155 }
10156 #[inline(always)]
10157 fn split_mask8x32(self, a: mask8x32<Self>) -> (mask8x16<Self>, mask8x16<Self>) {
10158 let bits = u64::from(a.val);
10159 (
10160 mask8x16 {
10161 val: (bits & 65535u64) as _,
10162 simd: self,
10163 },
10164 mask8x16 {
10165 val: ((bits >> 16usize) & 65535u64) as _,
10166 simd: self,
10167 },
10168 )
10169 }
10170 #[inline(always)]
10171 fn widen_mask8x32(self, a: mask8x32<Self>) -> (mask16x16<Self>, mask16x16<Self>) {
10172 let bits = u64::from(a.val);
10173 (
10174 mask16x16 {
10175 val: (bits & 65535u64) as _,
10176 simd: self,
10177 },
10178 mask16x16 {
10179 val: ((bits >> 16usize) & 65535u64) as _,
10180 simd: self,
10181 },
10182 )
10183 }
10184 #[inline(always)]
10185 fn abs_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
10186 crate::kernel!(
10187 #[inline(always)]
10188 fn kernel(token: Avx512, a: i16x16<Avx512>) -> i16x16<Avx512> {
10189 _mm256_abs_epi16(a.into()).simd_into(token)
10190 }
10191 );
10192 kernel(self, a)
10193 }
10194 #[inline(always)]
10195 fn splat_i16x16(self, val: i16) -> i16x16<Self> {
10196 crate::kernel!(
10197 #[inline(always)]
10198 fn kernel(token: Avx512, val: i16) -> i16x16<Avx512> {
10199 _mm256_set1_epi16(val).simd_into(token)
10200 }
10201 );
10202 kernel(self, val)
10203 }
10204 #[inline(always)]
10205 fn reverse_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
10206 let indices: u8x32<Self> = [
10207 30, 31, 28, 29, 26, 27, 24, 25, 22, 23, 20, 21, 18, 19, 16, 17, 14, 15, 12, 13, 10, 11,
10208 8, 9, 6, 7, 4, 5, 2, 3, 0, 1,
10209 ]
10210 .simd_into(self);
10211 self.swizzle_dyn_i16x16(a, indices)
10212 }
10213 #[inline(always)]
10214 fn slide_i16x16<const SHIFT: usize>(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10215 crate::kernel!(
10216 #[inline(always)]
10217 fn kernel(
10218 token: Avx512,
10219 a: i16x16<Avx512>,
10220 b: i16x16<Avx512>,
10221 shift: usize,
10222 ) -> i16x16<Avx512> {
10223 if shift >= 16usize {
10224 return b;
10225 }
10226 let idx = _mm256_add_epi8(
10227 _mm256_setr_epi8(
10228 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
10229 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
10230 ),
10231 _mm256_set1_epi8((shift * 2usize) as i8),
10232 );
10233 let result = _mm256_permutex2var_epi8(
10234 Bytes::to_bytes(a).val.0,
10235 idx,
10236 Bytes::to_bytes(b).val.0,
10237 );
10238 Bytes::from_bytes(u8x32 {
10239 val: crate::support::Aligned256(result),
10240 simd: token,
10241 })
10242 }
10243 );
10244 kernel(self, a, b, SHIFT)
10245 }
10246 #[inline(always)]
10247 fn slide_within_blocks_i16x16<const SHIFT: usize>(
10248 self,
10249 a: i16x16<Self>,
10250 b: i16x16<Self>,
10251 ) -> i16x16<Self> {
10252 if SHIFT == 0 {
10253 return a;
10254 }
10255 if SHIFT >= 8usize {
10256 return b;
10257 }
10258 let a = Bytes::to_bytes(a).val.0;
10259 let b = Bytes::to_bytes(b).val.0;
10260 let result = dyn_alignr_256(self, b, a, SHIFT * 2usize);
10261 Bytes::from_bytes(u8x32 {
10262 val: crate::support::Aligned256(result),
10263 simd: self,
10264 })
10265 }
10266 #[inline(always)]
10267 fn compress_i16x16(self, values: i16x16<Self>, mask: mask16x16<Self>) -> i16x16<Self> {
10268 crate::kernel!(
10269 #[inline(always)]
10270 fn kernel(
10271 token: Avx512,
10272 values: i16x16<Avx512>,
10273 mask: mask16x16<Avx512>,
10274 ) -> i16x16<Avx512> {
10275 _mm256_maskz_compress_epi16(u64::from((mask).val) as u16, values.into())
10276 .simd_into(token)
10277 }
10278 );
10279 kernel(self, values, mask)
10280 }
10281 #[inline(always)]
10282 fn compress_merge_i16x16(
10283 self,
10284 values: i16x16<Self>,
10285 mask: mask16x16<Self>,
10286 merge: i16x16<Self>,
10287 ) -> i16x16<Self> {
10288 crate::kernel!(
10289 #[inline(always)]
10290 fn kernel(
10291 token: Avx512,
10292 values: i16x16<Avx512>,
10293 mask: mask16x16<Avx512>,
10294 merge: i16x16<Avx512>,
10295 ) -> i16x16<Avx512> {
10296 _mm256_mask_compress_epi16(
10297 merge.into(),
10298 u64::from((mask).val) as u16,
10299 values.into(),
10300 )
10301 .simd_into(token)
10302 }
10303 );
10304 kernel(self, values, mask, merge)
10305 }
10306 #[inline(always)]
10307 fn expand_i16x16(self, values: i16x16<Self>, mask: mask16x16<Self>) -> i16x16<Self> {
10308 crate::kernel!(
10309 #[inline(always)]
10310 fn kernel(
10311 token: Avx512,
10312 values: i16x16<Avx512>,
10313 mask: mask16x16<Avx512>,
10314 ) -> i16x16<Avx512> {
10315 _mm256_maskz_expand_epi16(u64::from((mask).val) as u16, values.into())
10316 .simd_into(token)
10317 }
10318 );
10319 kernel(self, values, mask)
10320 }
10321 #[inline(always)]
10322 fn expand_merge_i16x16(
10323 self,
10324 values: i16x16<Self>,
10325 mask: mask16x16<Self>,
10326 merge: i16x16<Self>,
10327 ) -> i16x16<Self> {
10328 crate::kernel!(
10329 #[inline(always)]
10330 fn kernel(
10331 token: Avx512,
10332 values: i16x16<Avx512>,
10333 mask: mask16x16<Avx512>,
10334 merge: i16x16<Avx512>,
10335 ) -> i16x16<Avx512> {
10336 _mm256_mask_expand_epi16(merge.into(), u64::from((mask).val) as u16, values.into())
10337 .simd_into(token)
10338 }
10339 );
10340 kernel(self, values, mask, merge)
10341 }
10342 #[inline(always)]
10343 fn count_ones_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
10344 crate::kernel!(
10345 #[inline(always)]
10346 fn kernel(token: Avx512, a: i16x16<Avx512>) -> i16x16<Avx512> {
10347 _mm256_popcnt_epi16(a.into()).simd_into(token)
10348 }
10349 );
10350 kernel(self, a)
10351 }
10352 #[inline(always)]
10353 fn count_zeros_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
10354 self.count_ones_i16x16(self.not_i16x16(a))
10355 }
10356 #[inline(always)]
10357 fn add_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10358 crate::kernel!(
10359 #[inline(always)]
10360 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10361 _mm256_add_epi16(a.into(), b.into()).simd_into(token)
10362 }
10363 );
10364 kernel(self, a, b)
10365 }
10366 #[inline(always)]
10367 fn saturating_add_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10368 crate::kernel!(
10369 #[inline(always)]
10370 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10371 _mm256_adds_epi16(a.into(), b.into()).simd_into(token)
10372 }
10373 );
10374 kernel(self, a, b)
10375 }
10376 #[inline(always)]
10377 fn sub_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10378 crate::kernel!(
10379 #[inline(always)]
10380 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10381 _mm256_sub_epi16(a.into(), b.into()).simd_into(token)
10382 }
10383 );
10384 kernel(self, a, b)
10385 }
10386 #[inline(always)]
10387 fn saturating_sub_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10388 crate::kernel!(
10389 #[inline(always)]
10390 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10391 _mm256_subs_epi16(a.into(), b.into()).simd_into(token)
10392 }
10393 );
10394 kernel(self, a, b)
10395 }
10396 #[inline(always)]
10397 fn mul_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10398 crate::kernel!(
10399 #[inline(always)]
10400 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10401 _mm256_mullo_epi16(a.into(), b.into()).simd_into(token)
10402 }
10403 );
10404 kernel(self, a, b)
10405 }
10406 #[inline(always)]
10407 fn and_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10408 crate::kernel!(
10409 #[inline(always)]
10410 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10411 _mm256_and_si256(a.into(), b.into()).simd_into(token)
10412 }
10413 );
10414 kernel(self, a, b)
10415 }
10416 #[inline(always)]
10417 fn or_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10418 crate::kernel!(
10419 #[inline(always)]
10420 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10421 _mm256_or_si256(a.into(), b.into()).simd_into(token)
10422 }
10423 );
10424 kernel(self, a, b)
10425 }
10426 #[inline(always)]
10427 fn xor_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10428 crate::kernel!(
10429 #[inline(always)]
10430 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10431 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
10432 }
10433 );
10434 kernel(self, a, b)
10435 }
10436 #[inline(always)]
10437 fn not_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
10438 a ^ !0
10439 }
10440 #[inline(always)]
10441 fn shl_i16x16(self, a: i16x16<Self>, shift: u32) -> i16x16<Self> {
10442 crate::kernel!(
10443 #[inline(always)]
10444 fn kernel(token: Avx512, a: i16x16<Avx512>, shift: u32) -> i16x16<Avx512> {
10445 _mm256_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
10446 }
10447 );
10448 kernel(self, a, shift)
10449 }
10450 #[inline(always)]
10451 fn shlv_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10452 crate::kernel!(
10453 #[inline(always)]
10454 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10455 _mm256_sllv_epi16(a.into(), b.into()).simd_into(token)
10456 }
10457 );
10458 kernel(self, a, b)
10459 }
10460 #[inline(always)]
10461 fn shr_i16x16(self, a: i16x16<Self>, shift: u32) -> i16x16<Self> {
10462 crate::kernel!(
10463 #[inline(always)]
10464 fn kernel(token: Avx512, a: i16x16<Avx512>, shift: u32) -> i16x16<Avx512> {
10465 _mm256_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
10466 }
10467 );
10468 kernel(self, a, shift)
10469 }
10470 #[inline(always)]
10471 fn shrv_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10472 crate::kernel!(
10473 #[inline(always)]
10474 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10475 _mm256_srav_epi16(a.into(), b.into()).simd_into(token)
10476 }
10477 );
10478 kernel(self, a, b)
10479 }
10480 #[inline(always)]
10481 fn max_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10482 crate::kernel!(
10483 #[inline(always)]
10484 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10485 _mm256_max_epi16(a.into(), b.into()).simd_into(token)
10486 }
10487 );
10488 kernel(self, a, b)
10489 }
10490 #[inline(always)]
10491 fn min_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10492 crate::kernel!(
10493 #[inline(always)]
10494 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10495 _mm256_min_epi16(a.into(), b.into()).simd_into(token)
10496 }
10497 );
10498 kernel(self, a, b)
10499 }
10500 #[inline(always)]
10501 fn clamp_i16x16(self, a: i16x16<Self>, b: i16x16<Self>, c: i16x16<Self>) -> i16x16<Self> {
10502 debug_assert!(
10503 b.simd_le(c).all_true(),
10504 "clamp requires min <= max for all elements"
10505 );
10506 a.max(b).min(c)
10507 }
10508 #[inline(always)]
10509 fn simd_eq_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
10510 crate::kernel!(
10511 #[inline(always)]
10512 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> mask16x16<Avx512> {
10513 mask16x16 {
10514 val: _mm256_cmpeq_epi16_mask(a.into(), b.into()),
10515 simd: token,
10516 }
10517 }
10518 );
10519 kernel(self, a, b)
10520 }
10521 #[inline(always)]
10522 fn simd_ne_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
10523 crate::kernel!(
10524 #[inline(always)]
10525 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> mask16x16<Avx512> {
10526 mask16x16 {
10527 val: _mm256_cmpneq_epi16_mask(a.into(), b.into()),
10528 simd: token,
10529 }
10530 }
10531 );
10532 kernel(self, a, b)
10533 }
10534 #[inline(always)]
10535 fn simd_lt_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
10536 crate::kernel!(
10537 #[inline(always)]
10538 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> mask16x16<Avx512> {
10539 mask16x16 {
10540 val: _mm256_cmplt_epi16_mask(a.into(), b.into()),
10541 simd: token,
10542 }
10543 }
10544 );
10545 kernel(self, a, b)
10546 }
10547 #[inline(always)]
10548 fn simd_le_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
10549 crate::kernel!(
10550 #[inline(always)]
10551 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> mask16x16<Avx512> {
10552 mask16x16 {
10553 val: _mm256_cmple_epi16_mask(a.into(), b.into()),
10554 simd: token,
10555 }
10556 }
10557 );
10558 kernel(self, a, b)
10559 }
10560 #[inline(always)]
10561 fn zip_low_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10562 crate::kernel!(
10563 #[inline(always)]
10564 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10565 _mm256_permutex2var_epi16(
10566 a.into(),
10567 _mm256_setr_epi16(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
10568 b.into(),
10569 )
10570 .simd_into(token)
10571 }
10572 );
10573 kernel(self, a, b)
10574 }
10575 #[inline(always)]
10576 fn zip_high_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10577 crate::kernel!(
10578 #[inline(always)]
10579 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10580 _mm256_permutex2var_epi16(
10581 a.into(),
10582 _mm256_setr_epi16(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31),
10583 b.into(),
10584 )
10585 .simd_into(token)
10586 }
10587 );
10588 kernel(self, a, b)
10589 }
10590 #[inline(always)]
10591 fn unzip_low_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10592 crate::kernel!(
10593 #[inline(always)]
10594 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10595 _mm256_permutex2var_epi16(
10596 a.into(),
10597 _mm256_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
10598 b.into(),
10599 )
10600 .simd_into(token)
10601 }
10602 );
10603 kernel(self, a, b)
10604 }
10605 #[inline(always)]
10606 fn unzip_high_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10607 crate::kernel!(
10608 #[inline(always)]
10609 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
10610 _mm256_permutex2var_epi16(
10611 a.into(),
10612 _mm256_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
10613 b.into(),
10614 )
10615 .simd_into(token)
10616 }
10617 );
10618 kernel(self, a, b)
10619 }
10620 #[inline(always)]
10621 fn interleave_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i16x16<Self>, i16x16<Self>) {
10622 crate::kernel!(
10623 #[inline(always)]
10624 fn kernel(
10625 token: Avx512,
10626 a: i16x16<Avx512>,
10627 b: i16x16<Avx512>,
10628 ) -> (i16x16<Avx512>, i16x16<Avx512>) {
10629 let a = a.into();
10630 let b = b.into();
10631 (
10632 _mm256_permutex2var_epi16(
10633 a,
10634 _mm256_setr_epi16(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
10635 b,
10636 )
10637 .simd_into(token),
10638 _mm256_permutex2var_epi16(
10639 a,
10640 _mm256_setr_epi16(
10641 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31,
10642 ),
10643 b,
10644 )
10645 .simd_into(token),
10646 )
10647 }
10648 );
10649 kernel(self, a, b)
10650 }
10651 #[inline(always)]
10652 fn deinterleave_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i16x16<Self>, i16x16<Self>) {
10653 crate::kernel!(
10654 #[inline(always)]
10655 fn kernel(
10656 token: Avx512,
10657 a: i16x16<Avx512>,
10658 b: i16x16<Avx512>,
10659 ) -> (i16x16<Avx512>, i16x16<Avx512>) {
10660 let a = a.into();
10661 let b = b.into();
10662 (
10663 _mm256_permutex2var_epi16(
10664 a,
10665 _mm256_setr_epi16(
10666 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
10667 ),
10668 b,
10669 )
10670 .simd_into(token),
10671 _mm256_permutex2var_epi16(
10672 a,
10673 _mm256_setr_epi16(
10674 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31,
10675 ),
10676 b,
10677 )
10678 .simd_into(token),
10679 )
10680 }
10681 );
10682 kernel(self, a, b)
10683 }
10684 #[inline(always)]
10685 fn select_i16x16(self, a: mask16x16<Self>, b: i16x16<Self>, c: i16x16<Self>) -> i16x16<Self> {
10686 crate::kernel!(
10687 #[inline(always)]
10688 fn kernel(
10689 token: Avx512,
10690 a: mask16x16<Avx512>,
10691 b: i16x16<Avx512>,
10692 c: i16x16<Avx512>,
10693 ) -> i16x16<Avx512> {
10694 _mm256_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token)
10695 }
10696 );
10697 kernel(self, a, b, c)
10698 }
10699 #[inline(always)]
10700 fn combine_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x32<Self> {
10701 crate::kernel!(
10702 #[inline(always)]
10703 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x32<Avx512> {
10704 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
10705 }
10706 );
10707 kernel(self, a, b)
10708 }
10709 #[inline(always)]
10710 fn split_i16x16(self, a: i16x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
10711 crate::kernel!(
10712 #[inline(always)]
10713 fn kernel(token: Avx512, a: i16x16<Avx512>) -> (i16x8<Avx512>, i16x8<Avx512>) {
10714 (
10715 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
10716 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
10717 )
10718 }
10719 );
10720 kernel(self, a)
10721 }
10722 #[inline(always)]
10723 fn neg_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
10724 crate::kernel!(
10725 #[inline(always)]
10726 fn kernel(token: Avx512, a: i16x16<Avx512>) -> i16x16<Avx512> {
10727 _mm256_sub_epi16(_mm256_setzero_si256(), a.into()).simd_into(token)
10728 }
10729 );
10730 kernel(self, a)
10731 }
10732 #[inline(always)]
10733 fn widen_i16x16(self, a: i16x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
10734 crate::kernel!(
10735 #[inline(always)]
10736 fn kernel(token: Avx512, a: i16x16<Avx512>) -> (i32x8<Avx512>, i32x8<Avx512>) {
10737 let raw = a.into();
10738 (
10739 _mm256_cvtepi16_epi32(_mm256_castsi256_si128(raw)).simd_into(token),
10740 _mm256_cvtepi16_epi32(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
10741 )
10742 }
10743 );
10744 kernel(self, a)
10745 }
10746 #[inline(always)]
10747 fn widening_mul_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
10748 crate::kernel!(
10749 #[inline(always)]
10750 fn kernel(
10751 token: Avx512,
10752 a: i16x16<Avx512>,
10753 b: i16x16<Avx512>,
10754 ) -> (i32x8<Avx512>, i32x8<Avx512>) {
10755 let a = a.into();
10756 let b = b.into();
10757 let low = _mm256_mullo_epi16(a, b);
10758 let high = _mm256_mulhi_epi16(a, b);
10759 let result_low = _mm256_unpacklo_epi16(low, high);
10760 let result_high = _mm256_unpackhi_epi16(low, high);
10761 let lo = _mm256_permute2x128_si256::<0x20>(result_low, result_high);
10762 let hi = _mm256_permute2x128_si256::<0x31>(result_low, result_high);
10763 let result_low = lo;
10764 let result_high = hi;
10765 (result_low.simd_into(token), result_high.simd_into(token))
10766 }
10767 );
10768 kernel(self, a, b)
10769 }
10770 #[inline(always)]
10771 fn widening_add_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
10772 crate::kernel!(
10773 #[inline(always)]
10774 fn kernel(
10775 token: Avx512,
10776 a: i16x16<Avx512>,
10777 b: i16x16<Avx512>,
10778 ) -> (i32x8<Avx512>, i32x8<Avx512>) {
10779 let a = a.into();
10780 let b = b.into();
10781 let low = a;
10782 let high = b;
10783 let result_low = _mm256_unpacklo_epi16(low, high);
10784 let result_high = _mm256_unpackhi_epi16(low, high);
10785 let ones = _mm256_set1_epi16(1);
10786 let result_low = _mm256_madd_epi16(result_low, ones);
10787 let result_high = _mm256_madd_epi16(result_high, ones);
10788 let lo = _mm256_permute2x128_si256::<0x20>(result_low, result_high);
10789 let hi = _mm256_permute2x128_si256::<0x31>(result_low, result_high);
10790 let result_low = lo;
10791 let result_high = hi;
10792 (result_low.simd_into(token), result_high.simd_into(token))
10793 }
10794 );
10795 kernel(self, a, b)
10796 }
10797 #[inline(always)]
10798 fn pairwise_widening_add_i16x16(self, a: i16x16<Self>) -> i32x8<Self> {
10799 crate::kernel!(
10800 #[inline(always)]
10801 fn kernel(token: Avx512, a: i16x16<Avx512>) -> i32x8<Avx512> {
10802 ({ _mm256_madd_epi16(a.into(), _mm256_set1_epi16(1)) }).simd_into(token)
10803 }
10804 );
10805 kernel(self, a)
10806 }
10807 #[inline(always)]
10808 fn pairwise_dot_product_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i32x8<Self> {
10809 crate::kernel!(
10810 #[inline(always)]
10811 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i32x8<Avx512> {
10812 _mm256_madd_epi16(a.into(), b.into()).simd_into(token)
10813 }
10814 );
10815 kernel(self, a, b)
10816 }
10817 #[inline(always)]
10818 fn narrow_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i8x32<Self> {
10819 crate::kernel!(
10820 #[inline(always)]
10821 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i8x32<Avx512> {
10822 let low = _mm256_cvtepi16_epi8(a.into());
10823 let high = _mm256_cvtepi16_epi8(b.into());
10824 _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(low), high).simd_into(token)
10825 }
10826 );
10827 kernel(self, a, b)
10828 }
10829 #[inline(always)]
10830 fn saturating_narrow_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i8x32<Self> {
10831 crate::kernel!(
10832 #[inline(always)]
10833 fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i8x32<Avx512> {
10834 _mm256_permute4x64_epi64::<0xd8>(_mm256_packs_epi16(a.into(), b.into()))
10835 .simd_into(token)
10836 }
10837 );
10838 kernel(self, a, b)
10839 }
10840 #[inline(always)]
10841 fn relaxed_narrow_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i8x32<Self> {
10842 debug_assert!(
10843 a.as_slice()
10844 .iter()
10845 .chain(b.as_slice())
10846 .all(|&value| { value >= i8::MIN as i16 && value <= i8::MAX as i16 }),
10847 "relaxed_narrow inputs must fit in the destination type",
10848 );
10849 self.saturating_narrow_i16x16(a, b)
10850 }
10851 #[inline(always)]
10852 fn splat_u16x16(self, val: u16) -> u16x16<Self> {
10853 crate::kernel!(
10854 #[inline(always)]
10855 fn kernel(token: Avx512, val: u16) -> u16x16<Avx512> {
10856 _mm256_set1_epi16(val.cast_signed()).simd_into(token)
10857 }
10858 );
10859 kernel(self, val)
10860 }
10861 #[inline(always)]
10862 fn reverse_u16x16(self, a: u16x16<Self>) -> u16x16<Self> {
10863 let indices: u8x32<Self> = [
10864 30, 31, 28, 29, 26, 27, 24, 25, 22, 23, 20, 21, 18, 19, 16, 17, 14, 15, 12, 13, 10, 11,
10865 8, 9, 6, 7, 4, 5, 2, 3, 0, 1,
10866 ]
10867 .simd_into(self);
10868 self.swizzle_dyn_u16x16(a, indices)
10869 }
10870 #[inline(always)]
10871 fn slide_u16x16<const SHIFT: usize>(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
10872 crate::kernel!(
10873 #[inline(always)]
10874 fn kernel(
10875 token: Avx512,
10876 a: u16x16<Avx512>,
10877 b: u16x16<Avx512>,
10878 shift: usize,
10879 ) -> u16x16<Avx512> {
10880 if shift >= 16usize {
10881 return b;
10882 }
10883 let idx = _mm256_add_epi8(
10884 _mm256_setr_epi8(
10885 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
10886 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
10887 ),
10888 _mm256_set1_epi8((shift * 2usize) as i8),
10889 );
10890 let result = _mm256_permutex2var_epi8(
10891 Bytes::to_bytes(a).val.0,
10892 idx,
10893 Bytes::to_bytes(b).val.0,
10894 );
10895 Bytes::from_bytes(u8x32 {
10896 val: crate::support::Aligned256(result),
10897 simd: token,
10898 })
10899 }
10900 );
10901 kernel(self, a, b, SHIFT)
10902 }
10903 #[inline(always)]
10904 fn slide_within_blocks_u16x16<const SHIFT: usize>(
10905 self,
10906 a: u16x16<Self>,
10907 b: u16x16<Self>,
10908 ) -> u16x16<Self> {
10909 if SHIFT == 0 {
10910 return a;
10911 }
10912 if SHIFT >= 8usize {
10913 return b;
10914 }
10915 let a = Bytes::to_bytes(a).val.0;
10916 let b = Bytes::to_bytes(b).val.0;
10917 let result = dyn_alignr_256(self, b, a, SHIFT * 2usize);
10918 Bytes::from_bytes(u8x32 {
10919 val: crate::support::Aligned256(result),
10920 simd: self,
10921 })
10922 }
10923 #[inline(always)]
10924 fn compress_u16x16(self, values: u16x16<Self>, mask: mask16x16<Self>) -> u16x16<Self> {
10925 crate::kernel!(
10926 #[inline(always)]
10927 fn kernel(
10928 token: Avx512,
10929 values: u16x16<Avx512>,
10930 mask: mask16x16<Avx512>,
10931 ) -> u16x16<Avx512> {
10932 _mm256_maskz_compress_epi16(u64::from((mask).val) as u16, values.into())
10933 .simd_into(token)
10934 }
10935 );
10936 kernel(self, values, mask)
10937 }
10938 #[inline(always)]
10939 fn compress_merge_u16x16(
10940 self,
10941 values: u16x16<Self>,
10942 mask: mask16x16<Self>,
10943 merge: u16x16<Self>,
10944 ) -> u16x16<Self> {
10945 crate::kernel!(
10946 #[inline(always)]
10947 fn kernel(
10948 token: Avx512,
10949 values: u16x16<Avx512>,
10950 mask: mask16x16<Avx512>,
10951 merge: u16x16<Avx512>,
10952 ) -> u16x16<Avx512> {
10953 _mm256_mask_compress_epi16(
10954 merge.into(),
10955 u64::from((mask).val) as u16,
10956 values.into(),
10957 )
10958 .simd_into(token)
10959 }
10960 );
10961 kernel(self, values, mask, merge)
10962 }
10963 #[inline(always)]
10964 fn expand_u16x16(self, values: u16x16<Self>, mask: mask16x16<Self>) -> u16x16<Self> {
10965 crate::kernel!(
10966 #[inline(always)]
10967 fn kernel(
10968 token: Avx512,
10969 values: u16x16<Avx512>,
10970 mask: mask16x16<Avx512>,
10971 ) -> u16x16<Avx512> {
10972 _mm256_maskz_expand_epi16(u64::from((mask).val) as u16, values.into())
10973 .simd_into(token)
10974 }
10975 );
10976 kernel(self, values, mask)
10977 }
10978 #[inline(always)]
10979 fn expand_merge_u16x16(
10980 self,
10981 values: u16x16<Self>,
10982 mask: mask16x16<Self>,
10983 merge: u16x16<Self>,
10984 ) -> u16x16<Self> {
10985 crate::kernel!(
10986 #[inline(always)]
10987 fn kernel(
10988 token: Avx512,
10989 values: u16x16<Avx512>,
10990 mask: mask16x16<Avx512>,
10991 merge: u16x16<Avx512>,
10992 ) -> u16x16<Avx512> {
10993 _mm256_mask_expand_epi16(merge.into(), u64::from((mask).val) as u16, values.into())
10994 .simd_into(token)
10995 }
10996 );
10997 kernel(self, values, mask, merge)
10998 }
10999 #[inline(always)]
11000 fn count_ones_u16x16(self, a: u16x16<Self>) -> u16x16<Self> {
11001 crate::kernel!(
11002 #[inline(always)]
11003 fn kernel(token: Avx512, a: u16x16<Avx512>) -> u16x16<Avx512> {
11004 _mm256_popcnt_epi16(a.into()).simd_into(token)
11005 }
11006 );
11007 kernel(self, a)
11008 }
11009 #[inline(always)]
11010 fn count_zeros_u16x16(self, a: u16x16<Self>) -> u16x16<Self> {
11011 self.count_ones_u16x16(self.not_u16x16(a))
11012 }
11013 #[inline(always)]
11014 fn add_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11015 crate::kernel!(
11016 #[inline(always)]
11017 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11018 _mm256_add_epi16(a.into(), b.into()).simd_into(token)
11019 }
11020 );
11021 kernel(self, a, b)
11022 }
11023 #[inline(always)]
11024 fn saturating_add_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11025 crate::kernel!(
11026 #[inline(always)]
11027 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11028 _mm256_adds_epu16(a.into(), b.into()).simd_into(token)
11029 }
11030 );
11031 kernel(self, a, b)
11032 }
11033 #[inline(always)]
11034 fn sub_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11035 crate::kernel!(
11036 #[inline(always)]
11037 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11038 _mm256_sub_epi16(a.into(), b.into()).simd_into(token)
11039 }
11040 );
11041 kernel(self, a, b)
11042 }
11043 #[inline(always)]
11044 fn saturating_sub_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11045 crate::kernel!(
11046 #[inline(always)]
11047 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11048 _mm256_subs_epu16(a.into(), b.into()).simd_into(token)
11049 }
11050 );
11051 kernel(self, a, b)
11052 }
11053 #[inline(always)]
11054 fn mul_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11055 crate::kernel!(
11056 #[inline(always)]
11057 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11058 _mm256_mullo_epi16(a.into(), b.into()).simd_into(token)
11059 }
11060 );
11061 kernel(self, a, b)
11062 }
11063 #[inline(always)]
11064 fn and_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11065 crate::kernel!(
11066 #[inline(always)]
11067 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11068 _mm256_and_si256(a.into(), b.into()).simd_into(token)
11069 }
11070 );
11071 kernel(self, a, b)
11072 }
11073 #[inline(always)]
11074 fn or_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11075 crate::kernel!(
11076 #[inline(always)]
11077 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11078 _mm256_or_si256(a.into(), b.into()).simd_into(token)
11079 }
11080 );
11081 kernel(self, a, b)
11082 }
11083 #[inline(always)]
11084 fn xor_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11085 crate::kernel!(
11086 #[inline(always)]
11087 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11088 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
11089 }
11090 );
11091 kernel(self, a, b)
11092 }
11093 #[inline(always)]
11094 fn not_u16x16(self, a: u16x16<Self>) -> u16x16<Self> {
11095 a ^ !0
11096 }
11097 #[inline(always)]
11098 fn shl_u16x16(self, a: u16x16<Self>, shift: u32) -> u16x16<Self> {
11099 crate::kernel!(
11100 #[inline(always)]
11101 fn kernel(token: Avx512, a: u16x16<Avx512>, shift: u32) -> u16x16<Avx512> {
11102 _mm256_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
11103 }
11104 );
11105 kernel(self, a, shift)
11106 }
11107 #[inline(always)]
11108 fn shlv_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11109 crate::kernel!(
11110 #[inline(always)]
11111 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11112 _mm256_sllv_epi16(a.into(), b.into()).simd_into(token)
11113 }
11114 );
11115 kernel(self, a, b)
11116 }
11117 #[inline(always)]
11118 fn shr_u16x16(self, a: u16x16<Self>, shift: u32) -> u16x16<Self> {
11119 crate::kernel!(
11120 #[inline(always)]
11121 fn kernel(token: Avx512, a: u16x16<Avx512>, shift: u32) -> u16x16<Avx512> {
11122 _mm256_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
11123 }
11124 );
11125 kernel(self, a, shift)
11126 }
11127 #[inline(always)]
11128 fn shrv_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11129 crate::kernel!(
11130 #[inline(always)]
11131 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11132 _mm256_srlv_epi16(a.into(), b.into()).simd_into(token)
11133 }
11134 );
11135 kernel(self, a, b)
11136 }
11137 #[inline(always)]
11138 fn max_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11139 crate::kernel!(
11140 #[inline(always)]
11141 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11142 _mm256_max_epu16(a.into(), b.into()).simd_into(token)
11143 }
11144 );
11145 kernel(self, a, b)
11146 }
11147 #[inline(always)]
11148 fn min_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11149 crate::kernel!(
11150 #[inline(always)]
11151 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11152 _mm256_min_epu16(a.into(), b.into()).simd_into(token)
11153 }
11154 );
11155 kernel(self, a, b)
11156 }
11157 #[inline(always)]
11158 fn clamp_u16x16(self, a: u16x16<Self>, b: u16x16<Self>, c: u16x16<Self>) -> u16x16<Self> {
11159 debug_assert!(
11160 b.simd_le(c).all_true(),
11161 "clamp requires min <= max for all elements"
11162 );
11163 a.max(b).min(c)
11164 }
11165 #[inline(always)]
11166 fn simd_eq_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
11167 crate::kernel!(
11168 #[inline(always)]
11169 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> mask16x16<Avx512> {
11170 mask16x16 {
11171 val: _mm256_cmpeq_epu16_mask(a.into(), b.into()),
11172 simd: token,
11173 }
11174 }
11175 );
11176 kernel(self, a, b)
11177 }
11178 #[inline(always)]
11179 fn simd_ne_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
11180 crate::kernel!(
11181 #[inline(always)]
11182 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> mask16x16<Avx512> {
11183 mask16x16 {
11184 val: _mm256_cmpneq_epu16_mask(a.into(), b.into()),
11185 simd: token,
11186 }
11187 }
11188 );
11189 kernel(self, a, b)
11190 }
11191 #[inline(always)]
11192 fn simd_lt_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
11193 crate::kernel!(
11194 #[inline(always)]
11195 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> mask16x16<Avx512> {
11196 mask16x16 {
11197 val: _mm256_cmplt_epu16_mask(a.into(), b.into()),
11198 simd: token,
11199 }
11200 }
11201 );
11202 kernel(self, a, b)
11203 }
11204 #[inline(always)]
11205 fn simd_le_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
11206 crate::kernel!(
11207 #[inline(always)]
11208 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> mask16x16<Avx512> {
11209 mask16x16 {
11210 val: _mm256_cmple_epu16_mask(a.into(), b.into()),
11211 simd: token,
11212 }
11213 }
11214 );
11215 kernel(self, a, b)
11216 }
11217 #[inline(always)]
11218 fn zip_low_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11219 crate::kernel!(
11220 #[inline(always)]
11221 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11222 _mm256_permutex2var_epi16(
11223 a.into(),
11224 _mm256_setr_epi16(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
11225 b.into(),
11226 )
11227 .simd_into(token)
11228 }
11229 );
11230 kernel(self, a, b)
11231 }
11232 #[inline(always)]
11233 fn zip_high_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11234 crate::kernel!(
11235 #[inline(always)]
11236 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11237 _mm256_permutex2var_epi16(
11238 a.into(),
11239 _mm256_setr_epi16(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31),
11240 b.into(),
11241 )
11242 .simd_into(token)
11243 }
11244 );
11245 kernel(self, a, b)
11246 }
11247 #[inline(always)]
11248 fn unzip_low_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11249 crate::kernel!(
11250 #[inline(always)]
11251 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11252 _mm256_permutex2var_epi16(
11253 a.into(),
11254 _mm256_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
11255 b.into(),
11256 )
11257 .simd_into(token)
11258 }
11259 );
11260 kernel(self, a, b)
11261 }
11262 #[inline(always)]
11263 fn unzip_high_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11264 crate::kernel!(
11265 #[inline(always)]
11266 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
11267 _mm256_permutex2var_epi16(
11268 a.into(),
11269 _mm256_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
11270 b.into(),
11271 )
11272 .simd_into(token)
11273 }
11274 );
11275 kernel(self, a, b)
11276 }
11277 #[inline(always)]
11278 fn interleave_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u16x16<Self>, u16x16<Self>) {
11279 crate::kernel!(
11280 #[inline(always)]
11281 fn kernel(
11282 token: Avx512,
11283 a: u16x16<Avx512>,
11284 b: u16x16<Avx512>,
11285 ) -> (u16x16<Avx512>, u16x16<Avx512>) {
11286 let a = a.into();
11287 let b = b.into();
11288 (
11289 _mm256_permutex2var_epi16(
11290 a,
11291 _mm256_setr_epi16(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
11292 b,
11293 )
11294 .simd_into(token),
11295 _mm256_permutex2var_epi16(
11296 a,
11297 _mm256_setr_epi16(
11298 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31,
11299 ),
11300 b,
11301 )
11302 .simd_into(token),
11303 )
11304 }
11305 );
11306 kernel(self, a, b)
11307 }
11308 #[inline(always)]
11309 fn deinterleave_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u16x16<Self>, u16x16<Self>) {
11310 crate::kernel!(
11311 #[inline(always)]
11312 fn kernel(
11313 token: Avx512,
11314 a: u16x16<Avx512>,
11315 b: u16x16<Avx512>,
11316 ) -> (u16x16<Avx512>, u16x16<Avx512>) {
11317 let a = a.into();
11318 let b = b.into();
11319 (
11320 _mm256_permutex2var_epi16(
11321 a,
11322 _mm256_setr_epi16(
11323 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
11324 ),
11325 b,
11326 )
11327 .simd_into(token),
11328 _mm256_permutex2var_epi16(
11329 a,
11330 _mm256_setr_epi16(
11331 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31,
11332 ),
11333 b,
11334 )
11335 .simd_into(token),
11336 )
11337 }
11338 );
11339 kernel(self, a, b)
11340 }
11341 #[inline(always)]
11342 fn select_u16x16(self, a: mask16x16<Self>, b: u16x16<Self>, c: u16x16<Self>) -> u16x16<Self> {
11343 crate::kernel!(
11344 #[inline(always)]
11345 fn kernel(
11346 token: Avx512,
11347 a: mask16x16<Avx512>,
11348 b: u16x16<Avx512>,
11349 c: u16x16<Avx512>,
11350 ) -> u16x16<Avx512> {
11351 _mm256_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token)
11352 }
11353 );
11354 kernel(self, a, b, c)
11355 }
11356 #[inline(always)]
11357 fn combine_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x32<Self> {
11358 crate::kernel!(
11359 #[inline(always)]
11360 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x32<Avx512> {
11361 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
11362 }
11363 );
11364 kernel(self, a, b)
11365 }
11366 #[inline(always)]
11367 fn split_u16x16(self, a: u16x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
11368 crate::kernel!(
11369 #[inline(always)]
11370 fn kernel(token: Avx512, a: u16x16<Avx512>) -> (u16x8<Avx512>, u16x8<Avx512>) {
11371 (
11372 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
11373 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
11374 )
11375 }
11376 );
11377 kernel(self, a)
11378 }
11379 #[inline(always)]
11380 fn widen_u16x16(self, a: u16x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
11381 crate::kernel!(
11382 #[inline(always)]
11383 fn kernel(token: Avx512, a: u16x16<Avx512>) -> (u32x8<Avx512>, u32x8<Avx512>) {
11384 let raw = a.into();
11385 (
11386 _mm256_cvtepu16_epi32(_mm256_castsi256_si128(raw)).simd_into(token),
11387 _mm256_cvtepu16_epi32(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
11388 )
11389 }
11390 );
11391 kernel(self, a)
11392 }
11393 #[inline(always)]
11394 fn widening_mul_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
11395 crate::kernel!(
11396 #[inline(always)]
11397 fn kernel(
11398 token: Avx512,
11399 a: u16x16<Avx512>,
11400 b: u16x16<Avx512>,
11401 ) -> (u32x8<Avx512>, u32x8<Avx512>) {
11402 let a = a.into();
11403 let b = b.into();
11404 let low = _mm256_mullo_epi16(a, b);
11405 let high = _mm256_mulhi_epu16(a, b);
11406 let result_low = _mm256_unpacklo_epi16(low, high);
11407 let result_high = _mm256_unpackhi_epi16(low, high);
11408 let lo = _mm256_permute2x128_si256::<0x20>(result_low, result_high);
11409 let hi = _mm256_permute2x128_si256::<0x31>(result_low, result_high);
11410 let result_low = lo;
11411 let result_high = hi;
11412 (result_low.simd_into(token), result_high.simd_into(token))
11413 }
11414 );
11415 kernel(self, a, b)
11416 }
11417 #[inline(always)]
11418 fn widening_add_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
11419 let (a0, a1) = self.widen_u16x16(a);
11420 let (b0, b1) = self.widen_u16x16(b);
11421 (self.add_u32x8(a0, b0), self.add_u32x8(a1, b1))
11422 }
11423 #[inline(always)]
11424 fn pairwise_widening_add_u16x16(self, a: u16x16<Self>) -> u32x8<Self> {
11425 crate::kernel!(
11426 #[inline(always)]
11427 fn kernel(token: Avx512, a: u16x16<Avx512>) -> u32x8<Avx512> {
11428 ({
11429 let a = a.into();
11430 _mm256_add_epi32(
11431 _mm256_and_si256(a, _mm256_set1_epi32(65535)),
11432 _mm256_srli_epi32::<16>(a),
11433 )
11434 })
11435 .simd_into(token)
11436 }
11437 );
11438 kernel(self, a)
11439 }
11440 #[inline(always)]
11441 fn pairwise_dot_product_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u32x8<Self> {
11442 let (low, high) = self.widening_mul_u16x16(a, b);
11443 let (even, odd) = self.deinterleave_u32x8(low, high);
11444 self.add_u32x8(even, odd)
11445 }
11446 #[inline(always)]
11447 fn narrow_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u8x32<Self> {
11448 crate::kernel!(
11449 #[inline(always)]
11450 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u8x32<Avx512> {
11451 let low = _mm256_cvtepi16_epi8(a.into());
11452 let high = _mm256_cvtepi16_epi8(b.into());
11453 _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(low), high).simd_into(token)
11454 }
11455 );
11456 kernel(self, a, b)
11457 }
11458 #[inline(always)]
11459 fn saturating_narrow_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u8x32<Self> {
11460 crate::kernel!(
11461 #[inline(always)]
11462 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u8x32<Avx512> {
11463 _mm256_permute4x64_epi64::<0xd8>({
11464 let max = _mm256_set1_epi16(i16::MAX);
11465 _mm256_packus_epi16(
11466 _mm256_min_epu16(a.into(), max),
11467 _mm256_min_epu16(b.into(), max),
11468 )
11469 })
11470 .simd_into(token)
11471 }
11472 );
11473 kernel(self, a, b)
11474 }
11475 #[inline(always)]
11476 fn relaxed_narrow_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u8x32<Self> {
11477 crate::kernel!(
11478 #[inline(always)]
11479 fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u8x32<Avx512> {
11480 debug_assert!(
11481 a.as_slice()
11482 .iter()
11483 .chain(b.as_slice())
11484 .all(|&value| { value >= u8::MIN as u16 && value <= u8::MAX as u16 }),
11485 "relaxed_narrow inputs must fit in the destination type",
11486 );
11487 _mm256_permute4x64_epi64::<0xd8>(_mm256_packus_epi16(a.into(), b.into()))
11488 .simd_into(token)
11489 }
11490 );
11491 kernel(self, a, b)
11492 }
11493 #[inline(always)]
11494 fn splat_mask16x16(self, val: bool) -> mask16x16<Self> {
11495 mask16x16 {
11496 val: (if val { 65535u64 } else { 0 }) as _,
11497 simd: self,
11498 }
11499 }
11500 #[inline(always)]
11501 fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16<Self> {
11502 mask16x16 {
11503 val: (bits & 65535u64) as _,
11504 simd: self,
11505 }
11506 }
11507 #[inline(always)]
11508 fn to_bitmask_mask16x16(self, a: mask16x16<Self>) -> u64 {
11509 u64::from((a).val) & 65535u64
11510 }
11511 #[inline(always)]
11512 fn from_vector_mask16x16(self, vector: i16x16<Self>) -> mask16x16<Self> {
11513 crate::kernel!(
11514 #[inline(always)]
11515 fn kernel(token: Avx512, vector: i16x16<Avx512>) -> mask16x16<Avx512> {
11516 _mm256_movepi16_mask(vector.into()).simd_into(token)
11517 }
11518 );
11519 kernel(self, vector)
11520 }
11521 #[inline(always)]
11522 fn to_vector_mask16x16(self, a: mask16x16<Self>) -> i16x16<Self> {
11523 crate::kernel!(
11524 #[inline(always)]
11525 fn kernel(token: Avx512, a: mask16x16<Avx512>) -> i16x16<Avx512> {
11526 _mm256_movm_epi16(a.val).simd_into(token)
11527 }
11528 );
11529 kernel(self, a)
11530 }
11531 #[inline(always)]
11532 fn set_mask16x16(self, a: &mut mask16x16<Self>, index: usize, value: bool) -> () {
11533 assert!(
11534 index < 16usize,
11535 "mask lane index {index} is out of bounds for {} lanes",
11536 16usize
11537 );
11538 let bit = 1u64 << index;
11539 let bits = u64::from((a).val);
11540 let bits = if value { bits | bit } else { bits & !bit };
11541 *a = mask16x16 {
11542 val: (bits) as _,
11543 simd: self,
11544 };
11545 }
11546 #[inline(always)]
11547 fn rotate_elements_left_mask16x16<const OFFSET: usize>(
11548 self,
11549 a: mask16x16<Self>,
11550 ) -> mask16x16<Self> {
11551 mask16x16 {
11552 val: (a.val.rotate_right((OFFSET % 16) as u32)) as _,
11553 simd: self,
11554 }
11555 }
11556 #[inline(always)]
11557 fn rotate_elements_right_mask16x16<const OFFSET: usize>(
11558 self,
11559 a: mask16x16<Self>,
11560 ) -> mask16x16<Self> {
11561 mask16x16 {
11562 val: (a.val.rotate_left((OFFSET % 16) as u32)) as _,
11563 simd: self,
11564 }
11565 }
11566 #[inline(always)]
11567 fn and_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
11568 mask16x16 {
11569 val: ((u64::from((a).val) & u64::from((b).val)) & 65535u64) as _,
11570 simd: self,
11571 }
11572 }
11573 #[inline(always)]
11574 fn or_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
11575 mask16x16 {
11576 val: ((u64::from((a).val) | u64::from((b).val)) & 65535u64) as _,
11577 simd: self,
11578 }
11579 }
11580 #[inline(always)]
11581 fn xor_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
11582 mask16x16 {
11583 val: ((u64::from((a).val) ^ u64::from((b).val)) & 65535u64) as _,
11584 simd: self,
11585 }
11586 }
11587 #[inline(always)]
11588 fn not_mask16x16(self, a: mask16x16<Self>) -> mask16x16<Self> {
11589 mask16x16 {
11590 val: ((!u64::from((a).val)) & 65535u64) as _,
11591 simd: self,
11592 }
11593 }
11594 #[inline(always)]
11595 fn reverse_mask16x16(self, a: mask16x16<Self>) -> mask16x16<Self> {
11596 mask16x16 {
11597 val: (a.val.reverse_bits() >> 0usize) as _,
11598 simd: self,
11599 }
11600 }
11601 #[inline(always)]
11602 fn select_mask16x16(
11603 self,
11604 a: mask16x16<Self>,
11605 b: mask16x16<Self>,
11606 c: mask16x16<Self>,
11607 ) -> mask16x16<Self> {
11608 mask16x16 {
11609 val: (((u64::from((a).val) & u64::from((b).val))
11610 | ((!u64::from((a).val)) & u64::from((c).val)))
11611 & 65535u64) as _,
11612 simd: self,
11613 }
11614 }
11615 #[inline(always)]
11616 fn simd_eq_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
11617 mask16x16 {
11618 val: (!u64::from(a.val ^ b.val) & 65535u64) as _,
11619 simd: self,
11620 }
11621 }
11622 #[inline(always)]
11623 fn simd_ne_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
11624 mask16x16 {
11625 val: (u64::from(a.val ^ b.val) & 65535u64) as _,
11626 simd: self,
11627 }
11628 }
11629 #[inline(always)]
11630 fn any_true_mask16x16(self, a: mask16x16<Self>) -> bool {
11631 let bits = u64::from((a).val) & 65535u64;
11632 bits != 0
11633 }
11634 #[inline(always)]
11635 fn all_true_mask16x16(self, a: mask16x16<Self>) -> bool {
11636 let bits = u64::from((a).val) & 65535u64;
11637 bits == 65535u64
11638 }
11639 #[inline(always)]
11640 fn any_false_mask16x16(self, a: mask16x16<Self>) -> bool {
11641 let bits = u64::from((a).val) & 65535u64;
11642 bits != 65535u64
11643 }
11644 #[inline(always)]
11645 fn all_false_mask16x16(self, a: mask16x16<Self>) -> bool {
11646 let bits = u64::from((a).val) & 65535u64;
11647 bits == 0
11648 }
11649 #[inline(always)]
11650 fn combine_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x32<Self> {
11651 let bits = (u64::from(a.val) | (u64::from(b.val) << 16usize)) & 4294967295u64;
11652 mask16x32 {
11653 val: bits as _,
11654 simd: self,
11655 }
11656 }
11657 #[inline(always)]
11658 fn split_mask16x16(self, a: mask16x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
11659 let bits = u64::from(a.val);
11660 (
11661 mask16x8 {
11662 val: (bits & 255u64) as _,
11663 simd: self,
11664 },
11665 mask16x8 {
11666 val: ((bits >> 8usize) & 255u64) as _,
11667 simd: self,
11668 },
11669 )
11670 }
11671 #[inline(always)]
11672 fn widen_mask16x16(self, a: mask16x16<Self>) -> (mask32x8<Self>, mask32x8<Self>) {
11673 let bits = u64::from(a.val);
11674 (
11675 mask32x8 {
11676 val: (bits & 255u64) as _,
11677 simd: self,
11678 },
11679 mask32x8 {
11680 val: ((bits >> 8usize) & 255u64) as _,
11681 simd: self,
11682 },
11683 )
11684 }
11685 #[inline(always)]
11686 fn narrow_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask8x32<Self> {
11687 let low = u64::from(a.val) & 65535u64;
11688 let high = u64::from(b.val) & 65535u64;
11689 mask8x32 {
11690 val: (low | (high << 16usize)) as _,
11691 simd: self,
11692 }
11693 }
11694 #[inline(always)]
11695 fn abs_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
11696 crate::kernel!(
11697 #[inline(always)]
11698 fn kernel(token: Avx512, a: i32x8<Avx512>) -> i32x8<Avx512> {
11699 _mm256_abs_epi32(a.into()).simd_into(token)
11700 }
11701 );
11702 kernel(self, a)
11703 }
11704 #[inline(always)]
11705 fn splat_i32x8(self, val: i32) -> i32x8<Self> {
11706 crate::kernel!(
11707 #[inline(always)]
11708 fn kernel(token: Avx512, val: i32) -> i32x8<Avx512> {
11709 _mm256_set1_epi32(val).simd_into(token)
11710 }
11711 );
11712 kernel(self, val)
11713 }
11714 #[inline(always)]
11715 fn reverse_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
11716 let indices: u8x32<Self> = [
11717 28, 29, 30, 31, 24, 25, 26, 27, 20, 21, 22, 23, 16, 17, 18, 19, 12, 13, 14, 15, 8, 9,
11718 10, 11, 4, 5, 6, 7, 0, 1, 2, 3,
11719 ]
11720 .simd_into(self);
11721 self.swizzle_dyn_i32x8(a, indices)
11722 }
11723 #[inline(always)]
11724 fn slide_i32x8<const SHIFT: usize>(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11725 crate::kernel!(
11726 #[inline(always)]
11727 fn kernel(
11728 token: Avx512,
11729 a: i32x8<Avx512>,
11730 b: i32x8<Avx512>,
11731 shift: usize,
11732 ) -> i32x8<Avx512> {
11733 if shift >= 8usize {
11734 return b;
11735 }
11736 let idx = _mm256_add_epi8(
11737 _mm256_setr_epi8(
11738 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
11739 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
11740 ),
11741 _mm256_set1_epi8((shift * 4usize) as i8),
11742 );
11743 let result = _mm256_permutex2var_epi8(
11744 Bytes::to_bytes(a).val.0,
11745 idx,
11746 Bytes::to_bytes(b).val.0,
11747 );
11748 Bytes::from_bytes(u8x32 {
11749 val: crate::support::Aligned256(result),
11750 simd: token,
11751 })
11752 }
11753 );
11754 kernel(self, a, b, SHIFT)
11755 }
11756 #[inline(always)]
11757 fn slide_within_blocks_i32x8<const SHIFT: usize>(
11758 self,
11759 a: i32x8<Self>,
11760 b: i32x8<Self>,
11761 ) -> i32x8<Self> {
11762 if SHIFT == 0 {
11763 return a;
11764 }
11765 if SHIFT >= 4usize {
11766 return b;
11767 }
11768 let a = Bytes::to_bytes(a).val.0;
11769 let b = Bytes::to_bytes(b).val.0;
11770 let result = dyn_alignr_256(self, b, a, SHIFT * 4usize);
11771 Bytes::from_bytes(u8x32 {
11772 val: crate::support::Aligned256(result),
11773 simd: self,
11774 })
11775 }
11776 #[inline(always)]
11777 fn compress_i32x8(self, values: i32x8<Self>, mask: mask32x8<Self>) -> i32x8<Self> {
11778 crate::kernel!(
11779 #[inline(always)]
11780 fn kernel(
11781 token: Avx512,
11782 values: i32x8<Avx512>,
11783 mask: mask32x8<Avx512>,
11784 ) -> i32x8<Avx512> {
11785 _mm256_maskz_compress_epi32(u64::from((mask).val) as u8, values.into())
11786 .simd_into(token)
11787 }
11788 );
11789 kernel(self, values, mask)
11790 }
11791 #[inline(always)]
11792 fn compress_merge_i32x8(
11793 self,
11794 values: i32x8<Self>,
11795 mask: mask32x8<Self>,
11796 merge: i32x8<Self>,
11797 ) -> i32x8<Self> {
11798 crate::kernel!(
11799 #[inline(always)]
11800 fn kernel(
11801 token: Avx512,
11802 values: i32x8<Avx512>,
11803 mask: mask32x8<Avx512>,
11804 merge: i32x8<Avx512>,
11805 ) -> i32x8<Avx512> {
11806 _mm256_mask_compress_epi32(merge.into(), u64::from((mask).val) as u8, values.into())
11807 .simd_into(token)
11808 }
11809 );
11810 kernel(self, values, mask, merge)
11811 }
11812 #[inline(always)]
11813 fn expand_i32x8(self, values: i32x8<Self>, mask: mask32x8<Self>) -> i32x8<Self> {
11814 crate::kernel!(
11815 #[inline(always)]
11816 fn kernel(
11817 token: Avx512,
11818 values: i32x8<Avx512>,
11819 mask: mask32x8<Avx512>,
11820 ) -> i32x8<Avx512> {
11821 _mm256_maskz_expand_epi32(u64::from((mask).val) as u8, values.into())
11822 .simd_into(token)
11823 }
11824 );
11825 kernel(self, values, mask)
11826 }
11827 #[inline(always)]
11828 fn expand_merge_i32x8(
11829 self,
11830 values: i32x8<Self>,
11831 mask: mask32x8<Self>,
11832 merge: i32x8<Self>,
11833 ) -> i32x8<Self> {
11834 crate::kernel!(
11835 #[inline(always)]
11836 fn kernel(
11837 token: Avx512,
11838 values: i32x8<Avx512>,
11839 mask: mask32x8<Avx512>,
11840 merge: i32x8<Avx512>,
11841 ) -> i32x8<Avx512> {
11842 _mm256_mask_expand_epi32(merge.into(), u64::from((mask).val) as u8, values.into())
11843 .simd_into(token)
11844 }
11845 );
11846 kernel(self, values, mask, merge)
11847 }
11848 #[inline(always)]
11849 fn count_ones_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
11850 crate::kernel!(
11851 #[inline(always)]
11852 fn kernel(token: Avx512, a: i32x8<Avx512>) -> i32x8<Avx512> {
11853 _mm256_popcnt_epi32(a.into()).simd_into(token)
11854 }
11855 );
11856 kernel(self, a)
11857 }
11858 #[inline(always)]
11859 fn count_zeros_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
11860 self.count_ones_i32x8(self.not_i32x8(a))
11861 }
11862 #[inline(always)]
11863 fn add_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11864 crate::kernel!(
11865 #[inline(always)]
11866 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
11867 _mm256_add_epi32(a.into(), b.into()).simd_into(token)
11868 }
11869 );
11870 kernel(self, a, b)
11871 }
11872 #[inline(always)]
11873 fn saturating_add_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11874 crate::kernel!(
11875 #[inline(always)]
11876 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
11877 let a = a.into();
11878 let b = b.into();
11879 let wrapped = _mm256_add_epi32(a, b);
11880 let overflow_bits = _mm256_ternarylogic_epi32::<0x42>(a, b, wrapped);
11881 let overflow_mask = _mm256_srai_epi32::<31>(overflow_bits);
11882 let direction =
11883 _mm256_add_epi32(_mm256_srli_epi32::<31>(a), _mm256_set1_epi32(i32::MAX));
11884 _mm256_ternarylogic_epi32::<0xca>(overflow_mask, direction, wrapped)
11885 .simd_into(token)
11886 }
11887 );
11888 kernel(self, a, b)
11889 }
11890 #[inline(always)]
11891 fn sub_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11892 crate::kernel!(
11893 #[inline(always)]
11894 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
11895 _mm256_sub_epi32(a.into(), b.into()).simd_into(token)
11896 }
11897 );
11898 kernel(self, a, b)
11899 }
11900 #[inline(always)]
11901 fn saturating_sub_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11902 crate::kernel!(
11903 #[inline(always)]
11904 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
11905 let a = a.into();
11906 let b = b.into();
11907 let wrapped = _mm256_sub_epi32(a, b);
11908 let overflow_bits = _mm256_ternarylogic_epi32::<0x18>(a, b, wrapped);
11909 let overflow_mask = _mm256_srai_epi32::<31>(overflow_bits);
11910 let direction =
11911 _mm256_add_epi32(_mm256_srli_epi32::<31>(a), _mm256_set1_epi32(i32::MAX));
11912 _mm256_ternarylogic_epi32::<0xca>(overflow_mask, direction, wrapped)
11913 .simd_into(token)
11914 }
11915 );
11916 kernel(self, a, b)
11917 }
11918 #[inline(always)]
11919 fn mul_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11920 crate::kernel!(
11921 #[inline(always)]
11922 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
11923 _mm256_mullo_epi32(a.into(), b.into()).simd_into(token)
11924 }
11925 );
11926 kernel(self, a, b)
11927 }
11928 #[inline(always)]
11929 fn and_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11930 crate::kernel!(
11931 #[inline(always)]
11932 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
11933 _mm256_and_si256(a.into(), b.into()).simd_into(token)
11934 }
11935 );
11936 kernel(self, a, b)
11937 }
11938 #[inline(always)]
11939 fn or_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11940 crate::kernel!(
11941 #[inline(always)]
11942 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
11943 _mm256_or_si256(a.into(), b.into()).simd_into(token)
11944 }
11945 );
11946 kernel(self, a, b)
11947 }
11948 #[inline(always)]
11949 fn xor_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11950 crate::kernel!(
11951 #[inline(always)]
11952 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
11953 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
11954 }
11955 );
11956 kernel(self, a, b)
11957 }
11958 #[inline(always)]
11959 fn not_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
11960 a ^ !0
11961 }
11962 #[inline(always)]
11963 fn shl_i32x8(self, a: i32x8<Self>, shift: u32) -> i32x8<Self> {
11964 crate::kernel!(
11965 #[inline(always)]
11966 fn kernel(token: Avx512, a: i32x8<Avx512>, shift: u32) -> i32x8<Avx512> {
11967 _mm256_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
11968 }
11969 );
11970 kernel(self, a, shift)
11971 }
11972 #[inline(always)]
11973 fn shlv_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11974 crate::kernel!(
11975 #[inline(always)]
11976 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
11977 _mm256_sllv_epi32(a.into(), b.into()).simd_into(token)
11978 }
11979 );
11980 kernel(self, a, b)
11981 }
11982 #[inline(always)]
11983 fn shr_i32x8(self, a: i32x8<Self>, shift: u32) -> i32x8<Self> {
11984 crate::kernel!(
11985 #[inline(always)]
11986 fn kernel(token: Avx512, a: i32x8<Avx512>, shift: u32) -> i32x8<Avx512> {
11987 _mm256_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
11988 }
11989 );
11990 kernel(self, a, shift)
11991 }
11992 #[inline(always)]
11993 fn shrv_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11994 crate::kernel!(
11995 #[inline(always)]
11996 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
11997 _mm256_srav_epi32(a.into(), b.into()).simd_into(token)
11998 }
11999 );
12000 kernel(self, a, b)
12001 }
12002 #[inline(always)]
12003 fn max_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
12004 crate::kernel!(
12005 #[inline(always)]
12006 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
12007 _mm256_max_epi32(a.into(), b.into()).simd_into(token)
12008 }
12009 );
12010 kernel(self, a, b)
12011 }
12012 #[inline(always)]
12013 fn min_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
12014 crate::kernel!(
12015 #[inline(always)]
12016 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
12017 _mm256_min_epi32(a.into(), b.into()).simd_into(token)
12018 }
12019 );
12020 kernel(self, a, b)
12021 }
12022 #[inline(always)]
12023 fn clamp_i32x8(self, a: i32x8<Self>, b: i32x8<Self>, c: i32x8<Self>) -> i32x8<Self> {
12024 debug_assert!(
12025 b.simd_le(c).all_true(),
12026 "clamp requires min <= max for all elements"
12027 );
12028 a.max(b).min(c)
12029 }
12030 #[inline(always)]
12031 fn simd_eq_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
12032 crate::kernel!(
12033 #[inline(always)]
12034 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> mask32x8<Avx512> {
12035 mask32x8 {
12036 val: _mm256_cmpeq_epi32_mask(a.into(), b.into()),
12037 simd: token,
12038 }
12039 }
12040 );
12041 kernel(self, a, b)
12042 }
12043 #[inline(always)]
12044 fn simd_ne_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
12045 crate::kernel!(
12046 #[inline(always)]
12047 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> mask32x8<Avx512> {
12048 mask32x8 {
12049 val: _mm256_cmpneq_epi32_mask(a.into(), b.into()),
12050 simd: token,
12051 }
12052 }
12053 );
12054 kernel(self, a, b)
12055 }
12056 #[inline(always)]
12057 fn simd_lt_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
12058 crate::kernel!(
12059 #[inline(always)]
12060 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> mask32x8<Avx512> {
12061 mask32x8 {
12062 val: _mm256_cmplt_epi32_mask(a.into(), b.into()),
12063 simd: token,
12064 }
12065 }
12066 );
12067 kernel(self, a, b)
12068 }
12069 #[inline(always)]
12070 fn simd_le_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
12071 crate::kernel!(
12072 #[inline(always)]
12073 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> mask32x8<Avx512> {
12074 mask32x8 {
12075 val: _mm256_cmple_epi32_mask(a.into(), b.into()),
12076 simd: token,
12077 }
12078 }
12079 );
12080 kernel(self, a, b)
12081 }
12082 #[inline(always)]
12083 fn zip_low_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
12084 crate::kernel!(
12085 #[inline(always)]
12086 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
12087 _mm256_permutex2var_epi32(
12088 a.into(),
12089 _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11),
12090 b.into(),
12091 )
12092 .simd_into(token)
12093 }
12094 );
12095 kernel(self, a, b)
12096 }
12097 #[inline(always)]
12098 fn zip_high_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
12099 crate::kernel!(
12100 #[inline(always)]
12101 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
12102 _mm256_permutex2var_epi32(
12103 a.into(),
12104 _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15),
12105 b.into(),
12106 )
12107 .simd_into(token)
12108 }
12109 );
12110 kernel(self, a, b)
12111 }
12112 #[inline(always)]
12113 fn unzip_low_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
12114 crate::kernel!(
12115 #[inline(always)]
12116 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
12117 _mm256_permutex2var_epi32(
12118 a.into(),
12119 _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14),
12120 b.into(),
12121 )
12122 .simd_into(token)
12123 }
12124 );
12125 kernel(self, a, b)
12126 }
12127 #[inline(always)]
12128 fn unzip_high_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
12129 crate::kernel!(
12130 #[inline(always)]
12131 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
12132 _mm256_permutex2var_epi32(
12133 a.into(),
12134 _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15),
12135 b.into(),
12136 )
12137 .simd_into(token)
12138 }
12139 );
12140 kernel(self, a, b)
12141 }
12142 #[inline(always)]
12143 fn interleave_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i32x8<Self>, i32x8<Self>) {
12144 crate::kernel!(
12145 #[inline(always)]
12146 fn kernel(
12147 token: Avx512,
12148 a: i32x8<Avx512>,
12149 b: i32x8<Avx512>,
12150 ) -> (i32x8<Avx512>, i32x8<Avx512>) {
12151 let a = a.into();
12152 let b = b.into();
12153 (
12154 _mm256_permutex2var_epi32(a, _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11), b)
12155 .simd_into(token),
12156 _mm256_permutex2var_epi32(a, _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15), b)
12157 .simd_into(token),
12158 )
12159 }
12160 );
12161 kernel(self, a, b)
12162 }
12163 #[inline(always)]
12164 fn deinterleave_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i32x8<Self>, i32x8<Self>) {
12165 crate::kernel!(
12166 #[inline(always)]
12167 fn kernel(
12168 token: Avx512,
12169 a: i32x8<Avx512>,
12170 b: i32x8<Avx512>,
12171 ) -> (i32x8<Avx512>, i32x8<Avx512>) {
12172 let a = a.into();
12173 let b = b.into();
12174 (
12175 _mm256_permutex2var_epi32(a, _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14), b)
12176 .simd_into(token),
12177 _mm256_permutex2var_epi32(a, _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15), b)
12178 .simd_into(token),
12179 )
12180 }
12181 );
12182 kernel(self, a, b)
12183 }
12184 #[inline(always)]
12185 fn select_i32x8(self, a: mask32x8<Self>, b: i32x8<Self>, c: i32x8<Self>) -> i32x8<Self> {
12186 crate::kernel!(
12187 #[inline(always)]
12188 fn kernel(
12189 token: Avx512,
12190 a: mask32x8<Avx512>,
12191 b: i32x8<Avx512>,
12192 c: i32x8<Avx512>,
12193 ) -> i32x8<Avx512> {
12194 _mm256_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token)
12195 }
12196 );
12197 kernel(self, a, b, c)
12198 }
12199 #[inline(always)]
12200 fn combine_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x16<Self> {
12201 crate::kernel!(
12202 #[inline(always)]
12203 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x16<Avx512> {
12204 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
12205 }
12206 );
12207 kernel(self, a, b)
12208 }
12209 #[inline(always)]
12210 fn split_i32x8(self, a: i32x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
12211 crate::kernel!(
12212 #[inline(always)]
12213 fn kernel(token: Avx512, a: i32x8<Avx512>) -> (i32x4<Avx512>, i32x4<Avx512>) {
12214 (
12215 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
12216 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
12217 )
12218 }
12219 );
12220 kernel(self, a)
12221 }
12222 #[inline(always)]
12223 fn neg_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
12224 crate::kernel!(
12225 #[inline(always)]
12226 fn kernel(token: Avx512, a: i32x8<Avx512>) -> i32x8<Avx512> {
12227 _mm256_sub_epi32(_mm256_setzero_si256(), a.into()).simd_into(token)
12228 }
12229 );
12230 kernel(self, a)
12231 }
12232 #[inline(always)]
12233 fn widen_i32x8(self, a: i32x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
12234 crate::kernel!(
12235 #[inline(always)]
12236 fn kernel(token: Avx512, a: i32x8<Avx512>) -> (i64x4<Avx512>, i64x4<Avx512>) {
12237 let raw = a.into();
12238 (
12239 _mm256_cvtepi32_epi64(_mm256_castsi256_si128(raw)).simd_into(token),
12240 _mm256_cvtepi32_epi64(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
12241 )
12242 }
12243 );
12244 kernel(self, a)
12245 }
12246 #[inline(always)]
12247 fn widening_mul_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
12248 let (a0, a1) = self.widen_i32x8(a);
12249 let (b0, b1) = self.widen_i32x8(b);
12250 (self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1))
12251 }
12252 #[inline(always)]
12253 fn widening_add_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
12254 let (a0, a1) = self.widen_i32x8(a);
12255 let (b0, b1) = self.widen_i32x8(b);
12256 (self.add_i64x4(a0, b0), self.add_i64x4(a1, b1))
12257 }
12258 #[inline(always)]
12259 fn pairwise_widening_add_i32x8(self, a: i32x8<Self>) -> i64x4<Self> {
12260 crate::kernel!(
12261 #[inline(always)]
12262 fn kernel(token: Avx512, a: i32x8<Avx512>) -> i64x4<Avx512> {
12263 ({
12264 let y = _mm256_xor_si256(a.into(), _mm256_set1_epi32(i32::MIN));
12265 _mm256_add_epi64(
12266 _mm256_or_si256(y, _mm256_set1_epi64x(-0x1_0000_0000)),
12267 _mm256_srli_epi64::<32>(y),
12268 )
12269 })
12270 .simd_into(token)
12271 }
12272 );
12273 kernel(self, a)
12274 }
12275 #[inline(always)]
12276 fn pairwise_dot_product_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i64x4<Self> {
12277 let (low, high) = self.widening_mul_i32x8(a, b);
12278 let (even, odd) = self.deinterleave_i64x4(low, high);
12279 self.add_i64x4(even, odd)
12280 }
12281 #[inline(always)]
12282 fn narrow_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i16x16<Self> {
12283 crate::kernel!(
12284 #[inline(always)]
12285 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i16x16<Avx512> {
12286 let low = _mm256_cvtepi32_epi16(a.into());
12287 let high = _mm256_cvtepi32_epi16(b.into());
12288 _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(low), high).simd_into(token)
12289 }
12290 );
12291 kernel(self, a, b)
12292 }
12293 #[inline(always)]
12294 fn saturating_narrow_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i16x16<Self> {
12295 crate::kernel!(
12296 #[inline(always)]
12297 fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i16x16<Avx512> {
12298 _mm256_permute4x64_epi64::<0xd8>(_mm256_packs_epi32(a.into(), b.into()))
12299 .simd_into(token)
12300 }
12301 );
12302 kernel(self, a, b)
12303 }
12304 #[inline(always)]
12305 fn relaxed_narrow_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i16x16<Self> {
12306 debug_assert!(
12307 a.as_slice()
12308 .iter()
12309 .chain(b.as_slice())
12310 .all(|&value| { value >= i16::MIN as i32 && value <= i16::MAX as i32 }),
12311 "relaxed_narrow inputs must fit in the destination type",
12312 );
12313 self.saturating_narrow_i32x8(a, b)
12314 }
12315 #[inline(always)]
12316 fn cvt_f32_i32x8(self, a: i32x8<Self>) -> f32x8<Self> {
12317 crate::kernel!(
12318 #[inline(always)]
12319 fn kernel(token: Avx512, a: i32x8<Avx512>) -> f32x8<Avx512> {
12320 _mm256_cvtepi32_ps(a.into()).simd_into(token)
12321 }
12322 );
12323 kernel(self, a)
12324 }
12325 #[inline(always)]
12326 fn splat_u32x8(self, val: u32) -> u32x8<Self> {
12327 crate::kernel!(
12328 #[inline(always)]
12329 fn kernel(token: Avx512, val: u32) -> u32x8<Avx512> {
12330 _mm256_set1_epi32(val.cast_signed()).simd_into(token)
12331 }
12332 );
12333 kernel(self, val)
12334 }
12335 #[inline(always)]
12336 fn reverse_u32x8(self, a: u32x8<Self>) -> u32x8<Self> {
12337 let indices: u8x32<Self> = [
12338 28, 29, 30, 31, 24, 25, 26, 27, 20, 21, 22, 23, 16, 17, 18, 19, 12, 13, 14, 15, 8, 9,
12339 10, 11, 4, 5, 6, 7, 0, 1, 2, 3,
12340 ]
12341 .simd_into(self);
12342 self.swizzle_dyn_u32x8(a, indices)
12343 }
12344 #[inline(always)]
12345 fn slide_u32x8<const SHIFT: usize>(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12346 crate::kernel!(
12347 #[inline(always)]
12348 fn kernel(
12349 token: Avx512,
12350 a: u32x8<Avx512>,
12351 b: u32x8<Avx512>,
12352 shift: usize,
12353 ) -> u32x8<Avx512> {
12354 if shift >= 8usize {
12355 return b;
12356 }
12357 let idx = _mm256_add_epi8(
12358 _mm256_setr_epi8(
12359 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
12360 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
12361 ),
12362 _mm256_set1_epi8((shift * 4usize) as i8),
12363 );
12364 let result = _mm256_permutex2var_epi8(
12365 Bytes::to_bytes(a).val.0,
12366 idx,
12367 Bytes::to_bytes(b).val.0,
12368 );
12369 Bytes::from_bytes(u8x32 {
12370 val: crate::support::Aligned256(result),
12371 simd: token,
12372 })
12373 }
12374 );
12375 kernel(self, a, b, SHIFT)
12376 }
12377 #[inline(always)]
12378 fn slide_within_blocks_u32x8<const SHIFT: usize>(
12379 self,
12380 a: u32x8<Self>,
12381 b: u32x8<Self>,
12382 ) -> u32x8<Self> {
12383 if SHIFT == 0 {
12384 return a;
12385 }
12386 if SHIFT >= 4usize {
12387 return b;
12388 }
12389 let a = Bytes::to_bytes(a).val.0;
12390 let b = Bytes::to_bytes(b).val.0;
12391 let result = dyn_alignr_256(self, b, a, SHIFT * 4usize);
12392 Bytes::from_bytes(u8x32 {
12393 val: crate::support::Aligned256(result),
12394 simd: self,
12395 })
12396 }
12397 #[inline(always)]
12398 fn compress_u32x8(self, values: u32x8<Self>, mask: mask32x8<Self>) -> u32x8<Self> {
12399 crate::kernel!(
12400 #[inline(always)]
12401 fn kernel(
12402 token: Avx512,
12403 values: u32x8<Avx512>,
12404 mask: mask32x8<Avx512>,
12405 ) -> u32x8<Avx512> {
12406 _mm256_maskz_compress_epi32(u64::from((mask).val) as u8, values.into())
12407 .simd_into(token)
12408 }
12409 );
12410 kernel(self, values, mask)
12411 }
12412 #[inline(always)]
12413 fn compress_merge_u32x8(
12414 self,
12415 values: u32x8<Self>,
12416 mask: mask32x8<Self>,
12417 merge: u32x8<Self>,
12418 ) -> u32x8<Self> {
12419 crate::kernel!(
12420 #[inline(always)]
12421 fn kernel(
12422 token: Avx512,
12423 values: u32x8<Avx512>,
12424 mask: mask32x8<Avx512>,
12425 merge: u32x8<Avx512>,
12426 ) -> u32x8<Avx512> {
12427 _mm256_mask_compress_epi32(merge.into(), u64::from((mask).val) as u8, values.into())
12428 .simd_into(token)
12429 }
12430 );
12431 kernel(self, values, mask, merge)
12432 }
12433 #[inline(always)]
12434 fn expand_u32x8(self, values: u32x8<Self>, mask: mask32x8<Self>) -> u32x8<Self> {
12435 crate::kernel!(
12436 #[inline(always)]
12437 fn kernel(
12438 token: Avx512,
12439 values: u32x8<Avx512>,
12440 mask: mask32x8<Avx512>,
12441 ) -> u32x8<Avx512> {
12442 _mm256_maskz_expand_epi32(u64::from((mask).val) as u8, values.into())
12443 .simd_into(token)
12444 }
12445 );
12446 kernel(self, values, mask)
12447 }
12448 #[inline(always)]
12449 fn expand_merge_u32x8(
12450 self,
12451 values: u32x8<Self>,
12452 mask: mask32x8<Self>,
12453 merge: u32x8<Self>,
12454 ) -> u32x8<Self> {
12455 crate::kernel!(
12456 #[inline(always)]
12457 fn kernel(
12458 token: Avx512,
12459 values: u32x8<Avx512>,
12460 mask: mask32x8<Avx512>,
12461 merge: u32x8<Avx512>,
12462 ) -> u32x8<Avx512> {
12463 _mm256_mask_expand_epi32(merge.into(), u64::from((mask).val) as u8, values.into())
12464 .simd_into(token)
12465 }
12466 );
12467 kernel(self, values, mask, merge)
12468 }
12469 #[inline(always)]
12470 fn count_ones_u32x8(self, a: u32x8<Self>) -> u32x8<Self> {
12471 crate::kernel!(
12472 #[inline(always)]
12473 fn kernel(token: Avx512, a: u32x8<Avx512>) -> u32x8<Avx512> {
12474 _mm256_popcnt_epi32(a.into()).simd_into(token)
12475 }
12476 );
12477 kernel(self, a)
12478 }
12479 #[inline(always)]
12480 fn count_zeros_u32x8(self, a: u32x8<Self>) -> u32x8<Self> {
12481 self.count_ones_u32x8(self.not_u32x8(a))
12482 }
12483 #[inline(always)]
12484 fn add_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12485 crate::kernel!(
12486 #[inline(always)]
12487 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12488 _mm256_add_epi32(a.into(), b.into()).simd_into(token)
12489 }
12490 );
12491 kernel(self, a, b)
12492 }
12493 #[inline(always)]
12494 fn saturating_add_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12495 crate::kernel!(
12496 #[inline(always)]
12497 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12498 let a = a.into();
12499 let b = b.into();
12500 let threshold = _mm256_xor_si256(b, _mm256_set1_epi32(-1));
12501 _mm256_add_epi32(_mm256_min_epu32(a, threshold), b).simd_into(token)
12502 }
12503 );
12504 kernel(self, a, b)
12505 }
12506 #[inline(always)]
12507 fn sub_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12508 crate::kernel!(
12509 #[inline(always)]
12510 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12511 _mm256_sub_epi32(a.into(), b.into()).simd_into(token)
12512 }
12513 );
12514 kernel(self, a, b)
12515 }
12516 #[inline(always)]
12517 fn saturating_sub_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12518 crate::kernel!(
12519 #[inline(always)]
12520 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12521 let a = a.into();
12522 let b = b.into();
12523 _mm256_sub_epi32(_mm256_max_epu32(a, b), b).simd_into(token)
12524 }
12525 );
12526 kernel(self, a, b)
12527 }
12528 #[inline(always)]
12529 fn mul_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12530 crate::kernel!(
12531 #[inline(always)]
12532 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12533 _mm256_mullo_epi32(a.into(), b.into()).simd_into(token)
12534 }
12535 );
12536 kernel(self, a, b)
12537 }
12538 #[inline(always)]
12539 fn and_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12540 crate::kernel!(
12541 #[inline(always)]
12542 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12543 _mm256_and_si256(a.into(), b.into()).simd_into(token)
12544 }
12545 );
12546 kernel(self, a, b)
12547 }
12548 #[inline(always)]
12549 fn or_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12550 crate::kernel!(
12551 #[inline(always)]
12552 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12553 _mm256_or_si256(a.into(), b.into()).simd_into(token)
12554 }
12555 );
12556 kernel(self, a, b)
12557 }
12558 #[inline(always)]
12559 fn xor_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12560 crate::kernel!(
12561 #[inline(always)]
12562 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12563 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
12564 }
12565 );
12566 kernel(self, a, b)
12567 }
12568 #[inline(always)]
12569 fn not_u32x8(self, a: u32x8<Self>) -> u32x8<Self> {
12570 a ^ !0
12571 }
12572 #[inline(always)]
12573 fn shl_u32x8(self, a: u32x8<Self>, shift: u32) -> u32x8<Self> {
12574 crate::kernel!(
12575 #[inline(always)]
12576 fn kernel(token: Avx512, a: u32x8<Avx512>, shift: u32) -> u32x8<Avx512> {
12577 _mm256_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
12578 }
12579 );
12580 kernel(self, a, shift)
12581 }
12582 #[inline(always)]
12583 fn shlv_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12584 crate::kernel!(
12585 #[inline(always)]
12586 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12587 _mm256_sllv_epi32(a.into(), b.into()).simd_into(token)
12588 }
12589 );
12590 kernel(self, a, b)
12591 }
12592 #[inline(always)]
12593 fn shr_u32x8(self, a: u32x8<Self>, shift: u32) -> u32x8<Self> {
12594 crate::kernel!(
12595 #[inline(always)]
12596 fn kernel(token: Avx512, a: u32x8<Avx512>, shift: u32) -> u32x8<Avx512> {
12597 _mm256_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
12598 }
12599 );
12600 kernel(self, a, shift)
12601 }
12602 #[inline(always)]
12603 fn shrv_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12604 crate::kernel!(
12605 #[inline(always)]
12606 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12607 _mm256_srlv_epi32(a.into(), b.into()).simd_into(token)
12608 }
12609 );
12610 kernel(self, a, b)
12611 }
12612 #[inline(always)]
12613 fn max_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12614 crate::kernel!(
12615 #[inline(always)]
12616 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12617 _mm256_max_epu32(a.into(), b.into()).simd_into(token)
12618 }
12619 );
12620 kernel(self, a, b)
12621 }
12622 #[inline(always)]
12623 fn min_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12624 crate::kernel!(
12625 #[inline(always)]
12626 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12627 _mm256_min_epu32(a.into(), b.into()).simd_into(token)
12628 }
12629 );
12630 kernel(self, a, b)
12631 }
12632 #[inline(always)]
12633 fn clamp_u32x8(self, a: u32x8<Self>, b: u32x8<Self>, c: u32x8<Self>) -> u32x8<Self> {
12634 debug_assert!(
12635 b.simd_le(c).all_true(),
12636 "clamp requires min <= max for all elements"
12637 );
12638 a.max(b).min(c)
12639 }
12640 #[inline(always)]
12641 fn simd_eq_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
12642 crate::kernel!(
12643 #[inline(always)]
12644 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> mask32x8<Avx512> {
12645 mask32x8 {
12646 val: _mm256_cmpeq_epu32_mask(a.into(), b.into()),
12647 simd: token,
12648 }
12649 }
12650 );
12651 kernel(self, a, b)
12652 }
12653 #[inline(always)]
12654 fn simd_ne_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
12655 crate::kernel!(
12656 #[inline(always)]
12657 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> mask32x8<Avx512> {
12658 mask32x8 {
12659 val: _mm256_cmpneq_epu32_mask(a.into(), b.into()),
12660 simd: token,
12661 }
12662 }
12663 );
12664 kernel(self, a, b)
12665 }
12666 #[inline(always)]
12667 fn simd_lt_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
12668 crate::kernel!(
12669 #[inline(always)]
12670 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> mask32x8<Avx512> {
12671 mask32x8 {
12672 val: _mm256_cmplt_epu32_mask(a.into(), b.into()),
12673 simd: token,
12674 }
12675 }
12676 );
12677 kernel(self, a, b)
12678 }
12679 #[inline(always)]
12680 fn simd_le_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
12681 crate::kernel!(
12682 #[inline(always)]
12683 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> mask32x8<Avx512> {
12684 mask32x8 {
12685 val: _mm256_cmple_epu32_mask(a.into(), b.into()),
12686 simd: token,
12687 }
12688 }
12689 );
12690 kernel(self, a, b)
12691 }
12692 #[inline(always)]
12693 fn zip_low_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12694 crate::kernel!(
12695 #[inline(always)]
12696 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12697 _mm256_permutex2var_epi32(
12698 a.into(),
12699 _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11),
12700 b.into(),
12701 )
12702 .simd_into(token)
12703 }
12704 );
12705 kernel(self, a, b)
12706 }
12707 #[inline(always)]
12708 fn zip_high_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12709 crate::kernel!(
12710 #[inline(always)]
12711 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12712 _mm256_permutex2var_epi32(
12713 a.into(),
12714 _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15),
12715 b.into(),
12716 )
12717 .simd_into(token)
12718 }
12719 );
12720 kernel(self, a, b)
12721 }
12722 #[inline(always)]
12723 fn unzip_low_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12724 crate::kernel!(
12725 #[inline(always)]
12726 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12727 _mm256_permutex2var_epi32(
12728 a.into(),
12729 _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14),
12730 b.into(),
12731 )
12732 .simd_into(token)
12733 }
12734 );
12735 kernel(self, a, b)
12736 }
12737 #[inline(always)]
12738 fn unzip_high_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12739 crate::kernel!(
12740 #[inline(always)]
12741 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
12742 _mm256_permutex2var_epi32(
12743 a.into(),
12744 _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15),
12745 b.into(),
12746 )
12747 .simd_into(token)
12748 }
12749 );
12750 kernel(self, a, b)
12751 }
12752 #[inline(always)]
12753 fn interleave_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u32x8<Self>, u32x8<Self>) {
12754 crate::kernel!(
12755 #[inline(always)]
12756 fn kernel(
12757 token: Avx512,
12758 a: u32x8<Avx512>,
12759 b: u32x8<Avx512>,
12760 ) -> (u32x8<Avx512>, u32x8<Avx512>) {
12761 let a = a.into();
12762 let b = b.into();
12763 (
12764 _mm256_permutex2var_epi32(a, _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11), b)
12765 .simd_into(token),
12766 _mm256_permutex2var_epi32(a, _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15), b)
12767 .simd_into(token),
12768 )
12769 }
12770 );
12771 kernel(self, a, b)
12772 }
12773 #[inline(always)]
12774 fn deinterleave_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u32x8<Self>, u32x8<Self>) {
12775 crate::kernel!(
12776 #[inline(always)]
12777 fn kernel(
12778 token: Avx512,
12779 a: u32x8<Avx512>,
12780 b: u32x8<Avx512>,
12781 ) -> (u32x8<Avx512>, u32x8<Avx512>) {
12782 let a = a.into();
12783 let b = b.into();
12784 (
12785 _mm256_permutex2var_epi32(a, _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14), b)
12786 .simd_into(token),
12787 _mm256_permutex2var_epi32(a, _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15), b)
12788 .simd_into(token),
12789 )
12790 }
12791 );
12792 kernel(self, a, b)
12793 }
12794 #[inline(always)]
12795 fn select_u32x8(self, a: mask32x8<Self>, b: u32x8<Self>, c: u32x8<Self>) -> u32x8<Self> {
12796 crate::kernel!(
12797 #[inline(always)]
12798 fn kernel(
12799 token: Avx512,
12800 a: mask32x8<Avx512>,
12801 b: u32x8<Avx512>,
12802 c: u32x8<Avx512>,
12803 ) -> u32x8<Avx512> {
12804 _mm256_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token)
12805 }
12806 );
12807 kernel(self, a, b, c)
12808 }
12809 #[inline(always)]
12810 fn combine_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x16<Self> {
12811 crate::kernel!(
12812 #[inline(always)]
12813 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x16<Avx512> {
12814 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
12815 }
12816 );
12817 kernel(self, a, b)
12818 }
12819 #[inline(always)]
12820 fn split_u32x8(self, a: u32x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
12821 crate::kernel!(
12822 #[inline(always)]
12823 fn kernel(token: Avx512, a: u32x8<Avx512>) -> (u32x4<Avx512>, u32x4<Avx512>) {
12824 (
12825 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
12826 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
12827 )
12828 }
12829 );
12830 kernel(self, a)
12831 }
12832 #[inline(always)]
12833 fn widen_u32x8(self, a: u32x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
12834 crate::kernel!(
12835 #[inline(always)]
12836 fn kernel(token: Avx512, a: u32x8<Avx512>) -> (u64x4<Avx512>, u64x4<Avx512>) {
12837 let raw = a.into();
12838 (
12839 _mm256_cvtepu32_epi64(_mm256_castsi256_si128(raw)).simd_into(token),
12840 _mm256_cvtepu32_epi64(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
12841 )
12842 }
12843 );
12844 kernel(self, a)
12845 }
12846 #[inline(always)]
12847 fn widening_mul_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
12848 let (a0, a1) = self.widen_u32x8(a);
12849 let (b0, b1) = self.widen_u32x8(b);
12850 (self.mul_u64x4(a0, b0), self.mul_u64x4(a1, b1))
12851 }
12852 #[inline(always)]
12853 fn widening_add_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
12854 let (a0, a1) = self.widen_u32x8(a);
12855 let (b0, b1) = self.widen_u32x8(b);
12856 (self.add_u64x4(a0, b0), self.add_u64x4(a1, b1))
12857 }
12858 #[inline(always)]
12859 fn pairwise_widening_add_u32x8(self, a: u32x8<Self>) -> u64x4<Self> {
12860 crate::kernel!(
12861 #[inline(always)]
12862 fn kernel(token: Avx512, a: u32x8<Avx512>) -> u64x4<Avx512> {
12863 ({
12864 let a = a.into();
12865 _mm256_add_epi64(
12866 _mm256_and_si256(a, _mm256_set1_epi64x(4294967295)),
12867 _mm256_srli_epi64::<32>(a),
12868 )
12869 })
12870 .simd_into(token)
12871 }
12872 );
12873 kernel(self, a)
12874 }
12875 #[inline(always)]
12876 fn pairwise_dot_product_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u64x4<Self> {
12877 let (low, high) = self.widening_mul_u32x8(a, b);
12878 let (even, odd) = self.deinterleave_u64x4(low, high);
12879 self.add_u64x4(even, odd)
12880 }
12881 #[inline(always)]
12882 fn narrow_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u16x16<Self> {
12883 crate::kernel!(
12884 #[inline(always)]
12885 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u16x16<Avx512> {
12886 let low = _mm256_cvtepi32_epi16(a.into());
12887 let high = _mm256_cvtepi32_epi16(b.into());
12888 _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(low), high).simd_into(token)
12889 }
12890 );
12891 kernel(self, a, b)
12892 }
12893 #[inline(always)]
12894 fn saturating_narrow_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u16x16<Self> {
12895 crate::kernel!(
12896 #[inline(always)]
12897 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u16x16<Avx512> {
12898 _mm256_permute4x64_epi64::<0xd8>({
12899 let max = _mm256_set1_epi32(i32::MAX);
12900 _mm256_packus_epi32(
12901 _mm256_min_epu32(a.into(), max),
12902 _mm256_min_epu32(b.into(), max),
12903 )
12904 })
12905 .simd_into(token)
12906 }
12907 );
12908 kernel(self, a, b)
12909 }
12910 #[inline(always)]
12911 fn relaxed_narrow_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u16x16<Self> {
12912 crate::kernel!(
12913 #[inline(always)]
12914 fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u16x16<Avx512> {
12915 debug_assert!(
12916 a.as_slice()
12917 .iter()
12918 .chain(b.as_slice())
12919 .all(|&value| { value >= u16::MIN as u32 && value <= u16::MAX as u32 }),
12920 "relaxed_narrow inputs must fit in the destination type",
12921 );
12922 _mm256_permute4x64_epi64::<0xd8>(_mm256_packus_epi32(a.into(), b.into()))
12923 .simd_into(token)
12924 }
12925 );
12926 kernel(self, a, b)
12927 }
12928 #[inline(always)]
12929 fn cvt_f32_u32x8(self, a: u32x8<Self>) -> f32x8<Self> {
12930 crate::kernel!(
12931 #[inline(always)]
12932 fn kernel(token: Avx512, a: u32x8<Avx512>) -> f32x8<Avx512> {
12933 _mm512_castps512_ps256(_mm512_cvtepu32_ps(_mm512_zextsi256_si512(a.into())))
12934 .simd_into(token)
12935 }
12936 );
12937 kernel(self, a)
12938 }
12939 #[inline(always)]
12940 fn splat_mask32x8(self, val: bool) -> mask32x8<Self> {
12941 mask32x8 {
12942 val: (if val { 255u64 } else { 0 }) as _,
12943 simd: self,
12944 }
12945 }
12946 #[inline(always)]
12947 fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8<Self> {
12948 mask32x8 {
12949 val: (bits & 255u64) as _,
12950 simd: self,
12951 }
12952 }
12953 #[inline(always)]
12954 fn to_bitmask_mask32x8(self, a: mask32x8<Self>) -> u64 {
12955 u64::from((a).val) & 255u64
12956 }
12957 #[inline(always)]
12958 fn from_vector_mask32x8(self, vector: i32x8<Self>) -> mask32x8<Self> {
12959 crate::kernel!(
12960 #[inline(always)]
12961 fn kernel(token: Avx512, vector: i32x8<Avx512>) -> mask32x8<Avx512> {
12962 _mm256_movepi32_mask(vector.into()).simd_into(token)
12963 }
12964 );
12965 kernel(self, vector)
12966 }
12967 #[inline(always)]
12968 fn to_vector_mask32x8(self, a: mask32x8<Self>) -> i32x8<Self> {
12969 crate::kernel!(
12970 #[inline(always)]
12971 fn kernel(token: Avx512, a: mask32x8<Avx512>) -> i32x8<Avx512> {
12972 _mm256_movm_epi32(a.val).simd_into(token)
12973 }
12974 );
12975 kernel(self, a)
12976 }
12977 #[inline(always)]
12978 fn set_mask32x8(self, a: &mut mask32x8<Self>, index: usize, value: bool) -> () {
12979 assert!(
12980 index < 8usize,
12981 "mask lane index {index} is out of bounds for {} lanes",
12982 8usize
12983 );
12984 let bit = 1u64 << index;
12985 let bits = u64::from((a).val);
12986 let bits = if value { bits | bit } else { bits & !bit };
12987 *a = mask32x8 {
12988 val: (bits) as _,
12989 simd: self,
12990 };
12991 }
12992 #[inline(always)]
12993 fn rotate_elements_left_mask32x8<const OFFSET: usize>(
12994 self,
12995 a: mask32x8<Self>,
12996 ) -> mask32x8<Self> {
12997 mask32x8 {
12998 val: (a.val.rotate_right((OFFSET % 8) as u32)) as _,
12999 simd: self,
13000 }
13001 }
13002 #[inline(always)]
13003 fn rotate_elements_right_mask32x8<const OFFSET: usize>(
13004 self,
13005 a: mask32x8<Self>,
13006 ) -> mask32x8<Self> {
13007 mask32x8 {
13008 val: (a.val.rotate_left((OFFSET % 8) as u32)) as _,
13009 simd: self,
13010 }
13011 }
13012 #[inline(always)]
13013 fn and_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
13014 mask32x8 {
13015 val: ((u64::from((a).val) & u64::from((b).val)) & 255u64) as _,
13016 simd: self,
13017 }
13018 }
13019 #[inline(always)]
13020 fn or_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
13021 mask32x8 {
13022 val: ((u64::from((a).val) | u64::from((b).val)) & 255u64) as _,
13023 simd: self,
13024 }
13025 }
13026 #[inline(always)]
13027 fn xor_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
13028 mask32x8 {
13029 val: ((u64::from((a).val) ^ u64::from((b).val)) & 255u64) as _,
13030 simd: self,
13031 }
13032 }
13033 #[inline(always)]
13034 fn not_mask32x8(self, a: mask32x8<Self>) -> mask32x8<Self> {
13035 mask32x8 {
13036 val: ((!u64::from((a).val)) & 255u64) as _,
13037 simd: self,
13038 }
13039 }
13040 #[inline(always)]
13041 fn reverse_mask32x8(self, a: mask32x8<Self>) -> mask32x8<Self> {
13042 mask32x8 {
13043 val: (a.val.reverse_bits() >> 0usize) as _,
13044 simd: self,
13045 }
13046 }
13047 #[inline(always)]
13048 fn select_mask32x8(
13049 self,
13050 a: mask32x8<Self>,
13051 b: mask32x8<Self>,
13052 c: mask32x8<Self>,
13053 ) -> mask32x8<Self> {
13054 mask32x8 {
13055 val: (((u64::from((a).val) & u64::from((b).val))
13056 | ((!u64::from((a).val)) & u64::from((c).val)))
13057 & 255u64) as _,
13058 simd: self,
13059 }
13060 }
13061 #[inline(always)]
13062 fn simd_eq_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
13063 mask32x8 {
13064 val: (!u64::from(a.val ^ b.val) & 255u64) as _,
13065 simd: self,
13066 }
13067 }
13068 #[inline(always)]
13069 fn simd_ne_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
13070 mask32x8 {
13071 val: (u64::from(a.val ^ b.val) & 255u64) as _,
13072 simd: self,
13073 }
13074 }
13075 #[inline(always)]
13076 fn any_true_mask32x8(self, a: mask32x8<Self>) -> bool {
13077 let bits = u64::from((a).val) & 255u64;
13078 bits != 0
13079 }
13080 #[inline(always)]
13081 fn all_true_mask32x8(self, a: mask32x8<Self>) -> bool {
13082 let bits = u64::from((a).val) & 255u64;
13083 bits == 255u64
13084 }
13085 #[inline(always)]
13086 fn any_false_mask32x8(self, a: mask32x8<Self>) -> bool {
13087 let bits = u64::from((a).val) & 255u64;
13088 bits != 255u64
13089 }
13090 #[inline(always)]
13091 fn all_false_mask32x8(self, a: mask32x8<Self>) -> bool {
13092 let bits = u64::from((a).val) & 255u64;
13093 bits == 0
13094 }
13095 #[inline(always)]
13096 fn combine_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x16<Self> {
13097 let bits = (u64::from(a.val) | (u64::from(b.val) << 8usize)) & 65535u64;
13098 mask32x16 {
13099 val: bits as _,
13100 simd: self,
13101 }
13102 }
13103 #[inline(always)]
13104 fn split_mask32x8(self, a: mask32x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
13105 let bits = u64::from(a.val);
13106 (
13107 mask32x4 {
13108 val: (bits & 15u64) as _,
13109 simd: self,
13110 },
13111 mask32x4 {
13112 val: ((bits >> 4usize) & 15u64) as _,
13113 simd: self,
13114 },
13115 )
13116 }
13117 #[inline(always)]
13118 fn widen_mask32x8(self, a: mask32x8<Self>) -> (mask64x4<Self>, mask64x4<Self>) {
13119 let bits = u64::from(a.val);
13120 (
13121 mask64x4 {
13122 val: (bits & 15u64) as _,
13123 simd: self,
13124 },
13125 mask64x4 {
13126 val: ((bits >> 4usize) & 15u64) as _,
13127 simd: self,
13128 },
13129 )
13130 }
13131 #[inline(always)]
13132 fn narrow_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask16x16<Self> {
13133 let low = u64::from(a.val) & 255u64;
13134 let high = u64::from(b.val) & 255u64;
13135 mask16x16 {
13136 val: (low | (high << 8usize)) as _,
13137 simd: self,
13138 }
13139 }
13140 #[inline(always)]
13141 fn abs_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13142 crate::kernel!(
13143 #[inline(always)]
13144 fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
13145 _mm256_andnot_pd(_mm256_set1_pd(-0.0), a.into()).simd_into(token)
13146 }
13147 );
13148 kernel(self, a)
13149 }
13150 #[inline(always)]
13151 fn splat_f64x4(self, val: f64) -> f64x4<Self> {
13152 crate::kernel!(
13153 #[inline(always)]
13154 fn kernel(token: Avx512, val: f64) -> f64x4<Avx512> {
13155 _mm256_set1_pd(val).simd_into(token)
13156 }
13157 );
13158 kernel(self, val)
13159 }
13160 #[inline(always)]
13161 fn reverse_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13162 let indices: u8x32<Self> = [
13163 24, 25, 26, 27, 28, 29, 30, 31, 16, 17, 18, 19, 20, 21, 22, 23, 8, 9, 10, 11, 12, 13,
13164 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
13165 ]
13166 .simd_into(self);
13167 self.swizzle_dyn_f64x4(a, indices)
13168 }
13169 #[inline(always)]
13170 fn slide_f64x4<const SHIFT: usize>(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13171 crate::kernel!(
13172 #[inline(always)]
13173 fn kernel(
13174 token: Avx512,
13175 a: f64x4<Avx512>,
13176 b: f64x4<Avx512>,
13177 shift: usize,
13178 ) -> f64x4<Avx512> {
13179 if shift >= 4usize {
13180 return b;
13181 }
13182 let idx = _mm256_add_epi8(
13183 _mm256_setr_epi8(
13184 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
13185 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
13186 ),
13187 _mm256_set1_epi8((shift * 8usize) as i8),
13188 );
13189 let result = _mm256_permutex2var_epi8(
13190 Bytes::to_bytes(a).val.0,
13191 idx,
13192 Bytes::to_bytes(b).val.0,
13193 );
13194 Bytes::from_bytes(u8x32 {
13195 val: crate::support::Aligned256(result),
13196 simd: token,
13197 })
13198 }
13199 );
13200 kernel(self, a, b, SHIFT)
13201 }
13202 #[inline(always)]
13203 fn slide_within_blocks_f64x4<const SHIFT: usize>(
13204 self,
13205 a: f64x4<Self>,
13206 b: f64x4<Self>,
13207 ) -> f64x4<Self> {
13208 if SHIFT == 0 {
13209 return a;
13210 }
13211 if SHIFT >= 2usize {
13212 return b;
13213 }
13214 let a = Bytes::to_bytes(a).val.0;
13215 let b = Bytes::to_bytes(b).val.0;
13216 let result = dyn_alignr_256(self, b, a, SHIFT * 8usize);
13217 Bytes::from_bytes(u8x32 {
13218 val: crate::support::Aligned256(result),
13219 simd: self,
13220 })
13221 }
13222 #[inline(always)]
13223 fn compress_f64x4(self, values: f64x4<Self>, mask: mask64x4<Self>) -> f64x4<Self> {
13224 crate::kernel!(
13225 #[inline(always)]
13226 fn kernel(
13227 token: Avx512,
13228 values: f64x4<Avx512>,
13229 mask: mask64x4<Avx512>,
13230 ) -> f64x4<Avx512> {
13231 _mm256_maskz_compress_pd(u64::from((mask).val) as u8, values.into())
13232 .simd_into(token)
13233 }
13234 );
13235 kernel(self, values, mask)
13236 }
13237 #[inline(always)]
13238 fn compress_merge_f64x4(
13239 self,
13240 values: f64x4<Self>,
13241 mask: mask64x4<Self>,
13242 merge: f64x4<Self>,
13243 ) -> f64x4<Self> {
13244 crate::kernel!(
13245 #[inline(always)]
13246 fn kernel(
13247 token: Avx512,
13248 values: f64x4<Avx512>,
13249 mask: mask64x4<Avx512>,
13250 merge: f64x4<Avx512>,
13251 ) -> f64x4<Avx512> {
13252 _mm256_mask_compress_pd(merge.into(), u64::from((mask).val) as u8, values.into())
13253 .simd_into(token)
13254 }
13255 );
13256 kernel(self, values, mask, merge)
13257 }
13258 #[inline(always)]
13259 fn expand_f64x4(self, values: f64x4<Self>, mask: mask64x4<Self>) -> f64x4<Self> {
13260 crate::kernel!(
13261 #[inline(always)]
13262 fn kernel(
13263 token: Avx512,
13264 values: f64x4<Avx512>,
13265 mask: mask64x4<Avx512>,
13266 ) -> f64x4<Avx512> {
13267 _mm256_maskz_expand_pd(u64::from((mask).val) as u8, values.into()).simd_into(token)
13268 }
13269 );
13270 kernel(self, values, mask)
13271 }
13272 #[inline(always)]
13273 fn expand_merge_f64x4(
13274 self,
13275 values: f64x4<Self>,
13276 mask: mask64x4<Self>,
13277 merge: f64x4<Self>,
13278 ) -> f64x4<Self> {
13279 crate::kernel!(
13280 #[inline(always)]
13281 fn kernel(
13282 token: Avx512,
13283 values: f64x4<Avx512>,
13284 mask: mask64x4<Avx512>,
13285 merge: f64x4<Avx512>,
13286 ) -> f64x4<Avx512> {
13287 _mm256_mask_expand_pd(merge.into(), u64::from((mask).val) as u8, values.into())
13288 .simd_into(token)
13289 }
13290 );
13291 kernel(self, values, mask, merge)
13292 }
13293 #[inline(always)]
13294 fn neg_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13295 crate::kernel!(
13296 #[inline(always)]
13297 fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
13298 _mm256_xor_pd(a.into(), _mm256_set1_pd(-0.0)).simd_into(token)
13299 }
13300 );
13301 kernel(self, a)
13302 }
13303 #[inline(always)]
13304 fn sqrt_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13305 crate::kernel!(
13306 #[inline(always)]
13307 fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
13308 _mm256_sqrt_pd(a.into()).simd_into(token)
13309 }
13310 );
13311 kernel(self, a)
13312 }
13313 #[inline(always)]
13314 fn approximate_recip_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13315 crate::kernel!(
13316 #[inline(always)]
13317 fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
13318 _mm256_rcp14_pd(a.into()).simd_into(token)
13319 }
13320 );
13321 kernel(self, a)
13322 }
13323 #[inline(always)]
13324 fn add_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13325 crate::kernel!(
13326 #[inline(always)]
13327 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
13328 _mm256_add_pd(a.into(), b.into()).simd_into(token)
13329 }
13330 );
13331 kernel(self, a, b)
13332 }
13333 #[inline(always)]
13334 fn sub_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13335 crate::kernel!(
13336 #[inline(always)]
13337 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
13338 _mm256_sub_pd(a.into(), b.into()).simd_into(token)
13339 }
13340 );
13341 kernel(self, a, b)
13342 }
13343 #[inline(always)]
13344 fn mul_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13345 crate::kernel!(
13346 #[inline(always)]
13347 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
13348 _mm256_mul_pd(a.into(), b.into()).simd_into(token)
13349 }
13350 );
13351 kernel(self, a, b)
13352 }
13353 #[inline(always)]
13354 fn div_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13355 crate::kernel!(
13356 #[inline(always)]
13357 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
13358 _mm256_div_pd(a.into(), b.into()).simd_into(token)
13359 }
13360 );
13361 kernel(self, a, b)
13362 }
13363 #[inline(always)]
13364 fn copysign_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13365 crate::kernel!(
13366 #[inline(always)]
13367 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
13368 let mask = _mm256_set1_pd(-0.0);
13369 _mm256_or_pd(
13370 _mm256_and_pd(mask, b.into()),
13371 _mm256_andnot_pd(mask, a.into()),
13372 )
13373 .simd_into(token)
13374 }
13375 );
13376 kernel(self, a, b)
13377 }
13378 #[inline(always)]
13379 fn max_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13380 crate::kernel!(
13381 #[inline(always)]
13382 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
13383 _mm256_max_pd(a.into(), b.into()).simd_into(token)
13384 }
13385 );
13386 kernel(self, a, b)
13387 }
13388 #[inline(always)]
13389 fn min_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13390 crate::kernel!(
13391 #[inline(always)]
13392 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
13393 _mm256_min_pd(a.into(), b.into()).simd_into(token)
13394 }
13395 );
13396 kernel(self, a, b)
13397 }
13398 #[inline(always)]
13399 fn max_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13400 crate::kernel!(
13401 #[inline(always)]
13402 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
13403 _mm256_range_pd::<5i32>(a.into(), b.into()).simd_into(token)
13404 }
13405 );
13406 kernel(self, a, b)
13407 }
13408 #[inline(always)]
13409 fn min_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13410 crate::kernel!(
13411 #[inline(always)]
13412 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
13413 _mm256_range_pd::<4i32>(a.into(), b.into()).simd_into(token)
13414 }
13415 );
13416 kernel(self, a, b)
13417 }
13418 #[inline(always)]
13419 fn clamp_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
13420 crate::kernel!(
13421 #[inline(always)]
13422 fn kernel(
13423 token: Avx512,
13424 a: f64x4<Avx512>,
13425 b: f64x4<Avx512>,
13426 c: f64x4<Avx512>,
13427 ) -> f64x4<Avx512> {
13428 debug_assert!(
13429 b.simd_le(c).all_true(),
13430 "clamp requires min <= max for all elements"
13431 );
13432 _mm256_min_pd(c.into(), _mm256_max_pd(b.into(), a.into())).simd_into(token)
13433 }
13434 );
13435 kernel(self, a, b, c)
13436 }
13437 #[inline(always)]
13438 fn simd_eq_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
13439 crate::kernel!(
13440 #[inline(always)]
13441 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> mask64x4<Avx512> {
13442 mask64x4 {
13443 val: _mm256_cmp_pd_mask::<0i32>(a.into(), b.into()),
13444 simd: token,
13445 }
13446 }
13447 );
13448 kernel(self, a, b)
13449 }
13450 #[inline(always)]
13451 fn simd_ne_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
13452 crate::kernel!(
13453 #[inline(always)]
13454 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> mask64x4<Avx512> {
13455 mask64x4 {
13456 val: _mm256_cmp_pd_mask::<4i32>(a.into(), b.into()),
13457 simd: token,
13458 }
13459 }
13460 );
13461 kernel(self, a, b)
13462 }
13463 #[inline(always)]
13464 fn simd_lt_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
13465 crate::kernel!(
13466 #[inline(always)]
13467 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> mask64x4<Avx512> {
13468 mask64x4 {
13469 val: _mm256_cmp_pd_mask::<17i32>(a.into(), b.into()),
13470 simd: token,
13471 }
13472 }
13473 );
13474 kernel(self, a, b)
13475 }
13476 #[inline(always)]
13477 fn simd_le_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
13478 crate::kernel!(
13479 #[inline(always)]
13480 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> mask64x4<Avx512> {
13481 mask64x4 {
13482 val: _mm256_cmp_pd_mask::<18i32>(a.into(), b.into()),
13483 simd: token,
13484 }
13485 }
13486 );
13487 kernel(self, a, b)
13488 }
13489 #[inline(always)]
13490 fn zip_low_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13491 crate::kernel!(
13492 #[inline(always)]
13493 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
13494 _mm256_permutex2var_pd(a.into(), _mm256_setr_epi64x(0, 4, 1, 5), b.into())
13495 .simd_into(token)
13496 }
13497 );
13498 kernel(self, a, b)
13499 }
13500 #[inline(always)]
13501 fn zip_high_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13502 crate::kernel!(
13503 #[inline(always)]
13504 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
13505 _mm256_permutex2var_pd(a.into(), _mm256_setr_epi64x(2, 6, 3, 7), b.into())
13506 .simd_into(token)
13507 }
13508 );
13509 kernel(self, a, b)
13510 }
13511 #[inline(always)]
13512 fn unzip_low_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13513 crate::kernel!(
13514 #[inline(always)]
13515 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
13516 _mm256_permutex2var_pd(a.into(), _mm256_setr_epi64x(0, 2, 4, 6), b.into())
13517 .simd_into(token)
13518 }
13519 );
13520 kernel(self, a, b)
13521 }
13522 #[inline(always)]
13523 fn unzip_high_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13524 crate::kernel!(
13525 #[inline(always)]
13526 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
13527 _mm256_permutex2var_pd(a.into(), _mm256_setr_epi64x(1, 3, 5, 7), b.into())
13528 .simd_into(token)
13529 }
13530 );
13531 kernel(self, a, b)
13532 }
13533 #[inline(always)]
13534 fn interleave_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> (f64x4<Self>, f64x4<Self>) {
13535 crate::kernel!(
13536 #[inline(always)]
13537 fn kernel(
13538 token: Avx512,
13539 a: f64x4<Avx512>,
13540 b: f64x4<Avx512>,
13541 ) -> (f64x4<Avx512>, f64x4<Avx512>) {
13542 let a = a.into();
13543 let b = b.into();
13544 (
13545 _mm256_permutex2var_pd(a, _mm256_setr_epi64x(0, 4, 1, 5), b).simd_into(token),
13546 _mm256_permutex2var_pd(a, _mm256_setr_epi64x(2, 6, 3, 7), b).simd_into(token),
13547 )
13548 }
13549 );
13550 kernel(self, a, b)
13551 }
13552 #[inline(always)]
13553 fn deinterleave_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> (f64x4<Self>, f64x4<Self>) {
13554 crate::kernel!(
13555 #[inline(always)]
13556 fn kernel(
13557 token: Avx512,
13558 a: f64x4<Avx512>,
13559 b: f64x4<Avx512>,
13560 ) -> (f64x4<Avx512>, f64x4<Avx512>) {
13561 let a = a.into();
13562 let b = b.into();
13563 (
13564 _mm256_permutex2var_pd(a, _mm256_setr_epi64x(0, 2, 4, 6), b).simd_into(token),
13565 _mm256_permutex2var_pd(a, _mm256_setr_epi64x(1, 3, 5, 7), b).simd_into(token),
13566 )
13567 }
13568 );
13569 kernel(self, a, b)
13570 }
13571 #[inline(always)]
13572 fn mul_add_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
13573 crate::kernel!(
13574 #[inline(always)]
13575 fn kernel(
13576 token: Avx512,
13577 a: f64x4<Avx512>,
13578 b: f64x4<Avx512>,
13579 c: f64x4<Avx512>,
13580 ) -> f64x4<Avx512> {
13581 _mm256_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token)
13582 }
13583 );
13584 kernel(self, a, b, c)
13585 }
13586 #[inline(always)]
13587 fn mul_add_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
13588 self.mul_add_f64x4(a, b, c)
13589 }
13590 #[inline(always)]
13591 fn mul_sub_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
13592 crate::kernel!(
13593 #[inline(always)]
13594 fn kernel(
13595 token: Avx512,
13596 a: f64x4<Avx512>,
13597 b: f64x4<Avx512>,
13598 c: f64x4<Avx512>,
13599 ) -> f64x4<Avx512> {
13600 _mm256_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token)
13601 }
13602 );
13603 kernel(self, a, b, c)
13604 }
13605 #[inline(always)]
13606 fn mul_sub_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
13607 self.mul_sub_f64x4(a, b, c)
13608 }
13609 #[inline(always)]
13610 fn floor_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13611 crate::kernel!(
13612 #[inline(always)]
13613 fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
13614 _mm256_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
13615 .simd_into(token)
13616 }
13617 );
13618 kernel(self, a)
13619 }
13620 #[inline(always)]
13621 fn ceil_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13622 crate::kernel!(
13623 #[inline(always)]
13624 fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
13625 _mm256_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
13626 .simd_into(token)
13627 }
13628 );
13629 kernel(self, a)
13630 }
13631 #[inline(always)]
13632 fn round_ties_even_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13633 crate::kernel!(
13634 #[inline(always)]
13635 fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
13636 _mm256_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
13637 .simd_into(token)
13638 }
13639 );
13640 kernel(self, a)
13641 }
13642 #[inline(always)]
13643 fn round_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13644 let bias = self.splat_f64x4(const { f64::next_down(0.5) });
13645 (a + self.copysign_f64x4(bias, a)).trunc()
13646 }
13647 #[inline(always)]
13648 fn fract_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13649 a - self.trunc_f64x4(a)
13650 }
13651 #[inline(always)]
13652 fn trunc_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13653 crate::kernel!(
13654 #[inline(always)]
13655 fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
13656 _mm256_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
13657 .simd_into(token)
13658 }
13659 );
13660 kernel(self, a)
13661 }
13662 #[inline(always)]
13663 fn to_degrees_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13664 a * const { f64::to_degrees(1.0) }
13665 }
13666 #[inline(always)]
13667 fn to_radians_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13668 a * const { f64::to_radians(1.0) }
13669 }
13670 #[inline(always)]
13671 fn is_nan_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13672 crate::kernel!(
13673 #[inline(always)]
13674 fn kernel(token: Avx512, a: f64x4<Avx512>) -> mask64x4<Avx512> {
13675 mask64x4 {
13676 val: _mm256_fpclass_pd_mask::<129i32>(a.into()),
13677 simd: token,
13678 }
13679 }
13680 );
13681 kernel(self, a)
13682 }
13683 #[inline(always)]
13684 fn is_infinite_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13685 crate::kernel!(
13686 #[inline(always)]
13687 fn kernel(token: Avx512, a: f64x4<Avx512>) -> mask64x4<Avx512> {
13688 mask64x4 {
13689 val: _mm256_fpclass_pd_mask::<24i32>(a.into()),
13690 simd: token,
13691 }
13692 }
13693 );
13694 kernel(self, a)
13695 }
13696 #[inline(always)]
13697 fn is_finite_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13698 crate::kernel!(
13699 #[inline(always)]
13700 fn kernel(token: Avx512, a: f64x4<Avx512>) -> mask64x4<Avx512> {
13701 !(mask64x4 {
13702 val: _mm256_fpclass_pd_mask::<153i32>(a.into()),
13703 simd: token,
13704 })
13705 }
13706 );
13707 kernel(self, a)
13708 }
13709 #[inline(always)]
13710 fn is_subnormal_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13711 crate::kernel!(
13712 #[inline(always)]
13713 fn kernel(token: Avx512, a: f64x4<Avx512>) -> mask64x4<Avx512> {
13714 mask64x4 {
13715 val: _mm256_fpclass_pd_mask::<129i32>(_mm256_xor_pd(
13716 a.into(),
13717 _mm256_set1_pd(f64::INFINITY),
13718 )),
13719 simd: token,
13720 }
13721 }
13722 );
13723 kernel(self, a)
13724 }
13725 #[inline(always)]
13726 fn is_normal_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13727 crate::kernel!(
13728 #[inline(always)]
13729 fn kernel(token: Avx512, a: f64x4<Avx512>) -> mask64x4<Avx512> {
13730 !(mask64x4 {
13731 val: _mm256_fpclass_pd_mask::<191i32>(a.into()),
13732 simd: token,
13733 })
13734 }
13735 );
13736 kernel(self, a)
13737 }
13738 #[inline(always)]
13739 fn is_sign_positive_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13740 let i: i64x4<Self> = a.bitcast();
13741 (i & i64::MIN).simd_eq(0)
13742 }
13743 #[inline(always)]
13744 fn is_sign_negative_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13745 !a.is_sign_positive()
13746 }
13747 #[inline(always)]
13748 fn select_f64x4(self, a: mask64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
13749 crate::kernel!(
13750 #[inline(always)]
13751 fn kernel(
13752 token: Avx512,
13753 a: mask64x4<Avx512>,
13754 b: f64x4<Avx512>,
13755 c: f64x4<Avx512>,
13756 ) -> f64x4<Avx512> {
13757 _mm256_mask_blend_pd(a.val, c.into(), b.into()).simd_into(token)
13758 }
13759 );
13760 kernel(self, a, b, c)
13761 }
13762 #[inline(always)]
13763 fn combine_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x8<Self> {
13764 crate::kernel!(
13765 #[inline(always)]
13766 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x8<Avx512> {
13767 _mm512_insertf64x4::<1>(_mm512_castpd256_pd512(a.into()), b.into()).simd_into(token)
13768 }
13769 );
13770 kernel(self, a, b)
13771 }
13772 #[inline(always)]
13773 fn split_f64x4(self, a: f64x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
13774 crate::kernel!(
13775 #[inline(always)]
13776 fn kernel(token: Avx512, a: f64x4<Avx512>) -> (f64x2<Avx512>, f64x2<Avx512>) {
13777 (
13778 _mm256_extractf128_pd::<0>(a.into()).simd_into(token),
13779 _mm256_extractf128_pd::<1>(a.into()).simd_into(token),
13780 )
13781 }
13782 );
13783 kernel(self, a)
13784 }
13785 #[inline(always)]
13786 fn narrow_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f32x8<Self> {
13787 crate::kernel!(
13788 #[inline(always)]
13789 fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f32x8<Avx512> {
13790 let low = _mm256_cvtpd_ps(a.into());
13791 let high = _mm256_cvtpd_ps(b.into());
13792 _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(low), high).simd_into(token)
13793 }
13794 );
13795 kernel(self, a, b)
13796 }
13797 #[inline(always)]
13798 fn saturating_narrow_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f32x8<Self> {
13799 self.narrow_f64x4(a, b)
13800 }
13801 #[inline(always)]
13802 fn relaxed_narrow_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f32x8<Self> {
13803 self.narrow_f64x4(a, b)
13804 }
13805 #[inline(always)]
13806 fn cvt_u64_f64x4(self, a: f64x4<Self>) -> u64x4<Self> {
13807 crate::kernel!(
13808 #[inline(always)]
13809 fn kernel(token: Avx512, a: f64x4<Avx512>) -> u64x4<Avx512> {
13810 _mm256_cvttpd_epu64(a.into()).simd_into(token)
13811 }
13812 );
13813 kernel(self, a)
13814 }
13815 #[inline(always)]
13816 fn cvt_u64_precise_f64x4(self, a: f64x4<Self>) -> u64x4<Self> {
13817 crate::kernel!(
13818 #[inline(always)]
13819 fn kernel(token: Avx512, a: f64x4<Avx512>) -> u64x4<Avx512> {
13820 let a = a.into();
13821 let positive = _mm256_cmp_pd_mask::<17i32>(_mm256_setzero_pd(), a);
13822 _mm256_maskz_cvttpd_epu64(positive, a).simd_into(token)
13823 }
13824 );
13825 kernel(self, a)
13826 }
13827 #[inline(always)]
13828 fn cvt_i64_f64x4(self, a: f64x4<Self>) -> i64x4<Self> {
13829 crate::kernel!(
13830 #[inline(always)]
13831 fn kernel(token: Avx512, a: f64x4<Avx512>) -> i64x4<Avx512> {
13832 _mm256_cvttpd_epi64(a.into()).simd_into(token)
13833 }
13834 );
13835 kernel(self, a)
13836 }
13837 #[inline(always)]
13838 fn cvt_i64_precise_f64x4(self, a: f64x4<Self>) -> i64x4<Self> {
13839 crate::kernel!(
13840 #[inline(always)]
13841 fn kernel(token: Avx512, a: f64x4<Avx512>) -> i64x4<Avx512> {
13842 let a = a.into();
13843 let in_range =
13844 _mm256_cmp_pd_mask::<17i32>(a, _mm256_set1_pd(9223372036854775808.0));
13845 let mut converted =
13846 _mm256_mask_cvttpd_epi64(_mm256_set1_epi64x(i64::MAX), in_range, a);
13847 let is_not_nan = _mm256_cmp_pd_mask::<7i32>(a, a);
13848 converted = _mm256_mask_blend_epi64(is_not_nan, _mm256_setzero_si256(), converted);
13849 converted.simd_into(token)
13850 }
13851 );
13852 kernel(self, a)
13853 }
13854 #[inline(always)]
13855 fn abs_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
13856 crate::kernel!(
13857 #[inline(always)]
13858 fn kernel(token: Avx512, a: i64x4<Avx512>) -> i64x4<Avx512> {
13859 _mm256_abs_epi64(a.into()).simd_into(token)
13860 }
13861 );
13862 kernel(self, a)
13863 }
13864 #[inline(always)]
13865 fn splat_i64x4(self, val: i64) -> i64x4<Self> {
13866 crate::kernel!(
13867 #[inline(always)]
13868 fn kernel(token: Avx512, val: i64) -> i64x4<Avx512> {
13869 _mm256_set1_epi64x(val).simd_into(token)
13870 }
13871 );
13872 kernel(self, val)
13873 }
13874 #[inline(always)]
13875 fn reverse_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
13876 let indices: u8x32<Self> = [
13877 24, 25, 26, 27, 28, 29, 30, 31, 16, 17, 18, 19, 20, 21, 22, 23, 8, 9, 10, 11, 12, 13,
13878 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
13879 ]
13880 .simd_into(self);
13881 self.swizzle_dyn_i64x4(a, indices)
13882 }
13883 #[inline(always)]
13884 fn slide_i64x4<const SHIFT: usize>(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
13885 crate::kernel!(
13886 #[inline(always)]
13887 fn kernel(
13888 token: Avx512,
13889 a: i64x4<Avx512>,
13890 b: i64x4<Avx512>,
13891 shift: usize,
13892 ) -> i64x4<Avx512> {
13893 if shift >= 4usize {
13894 return b;
13895 }
13896 let idx = _mm256_add_epi8(
13897 _mm256_setr_epi8(
13898 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
13899 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
13900 ),
13901 _mm256_set1_epi8((shift * 8usize) as i8),
13902 );
13903 let result = _mm256_permutex2var_epi8(
13904 Bytes::to_bytes(a).val.0,
13905 idx,
13906 Bytes::to_bytes(b).val.0,
13907 );
13908 Bytes::from_bytes(u8x32 {
13909 val: crate::support::Aligned256(result),
13910 simd: token,
13911 })
13912 }
13913 );
13914 kernel(self, a, b, SHIFT)
13915 }
13916 #[inline(always)]
13917 fn slide_within_blocks_i64x4<const SHIFT: usize>(
13918 self,
13919 a: i64x4<Self>,
13920 b: i64x4<Self>,
13921 ) -> i64x4<Self> {
13922 if SHIFT == 0 {
13923 return a;
13924 }
13925 if SHIFT >= 2usize {
13926 return b;
13927 }
13928 let a = Bytes::to_bytes(a).val.0;
13929 let b = Bytes::to_bytes(b).val.0;
13930 let result = dyn_alignr_256(self, b, a, SHIFT * 8usize);
13931 Bytes::from_bytes(u8x32 {
13932 val: crate::support::Aligned256(result),
13933 simd: self,
13934 })
13935 }
13936 #[inline(always)]
13937 fn compress_i64x4(self, values: i64x4<Self>, mask: mask64x4<Self>) -> i64x4<Self> {
13938 crate::kernel!(
13939 #[inline(always)]
13940 fn kernel(
13941 token: Avx512,
13942 values: i64x4<Avx512>,
13943 mask: mask64x4<Avx512>,
13944 ) -> i64x4<Avx512> {
13945 _mm256_maskz_compress_epi64(u64::from((mask).val) as u8, values.into())
13946 .simd_into(token)
13947 }
13948 );
13949 kernel(self, values, mask)
13950 }
13951 #[inline(always)]
13952 fn compress_merge_i64x4(
13953 self,
13954 values: i64x4<Self>,
13955 mask: mask64x4<Self>,
13956 merge: i64x4<Self>,
13957 ) -> i64x4<Self> {
13958 crate::kernel!(
13959 #[inline(always)]
13960 fn kernel(
13961 token: Avx512,
13962 values: i64x4<Avx512>,
13963 mask: mask64x4<Avx512>,
13964 merge: i64x4<Avx512>,
13965 ) -> i64x4<Avx512> {
13966 _mm256_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into())
13967 .simd_into(token)
13968 }
13969 );
13970 kernel(self, values, mask, merge)
13971 }
13972 #[inline(always)]
13973 fn expand_i64x4(self, values: i64x4<Self>, mask: mask64x4<Self>) -> i64x4<Self> {
13974 crate::kernel!(
13975 #[inline(always)]
13976 fn kernel(
13977 token: Avx512,
13978 values: i64x4<Avx512>,
13979 mask: mask64x4<Avx512>,
13980 ) -> i64x4<Avx512> {
13981 _mm256_maskz_expand_epi64(u64::from((mask).val) as u8, values.into())
13982 .simd_into(token)
13983 }
13984 );
13985 kernel(self, values, mask)
13986 }
13987 #[inline(always)]
13988 fn expand_merge_i64x4(
13989 self,
13990 values: i64x4<Self>,
13991 mask: mask64x4<Self>,
13992 merge: i64x4<Self>,
13993 ) -> i64x4<Self> {
13994 crate::kernel!(
13995 #[inline(always)]
13996 fn kernel(
13997 token: Avx512,
13998 values: i64x4<Avx512>,
13999 mask: mask64x4<Avx512>,
14000 merge: i64x4<Avx512>,
14001 ) -> i64x4<Avx512> {
14002 _mm256_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into())
14003 .simd_into(token)
14004 }
14005 );
14006 kernel(self, values, mask, merge)
14007 }
14008 #[inline(always)]
14009 fn count_ones_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
14010 crate::kernel!(
14011 #[inline(always)]
14012 fn kernel(token: Avx512, a: i64x4<Avx512>) -> i64x4<Avx512> {
14013 _mm256_popcnt_epi64(a.into()).simd_into(token)
14014 }
14015 );
14016 kernel(self, a)
14017 }
14018 #[inline(always)]
14019 fn count_zeros_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
14020 self.count_ones_i64x4(self.not_i64x4(a))
14021 }
14022 #[inline(always)]
14023 fn add_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14024 crate::kernel!(
14025 #[inline(always)]
14026 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14027 _mm256_add_epi64(a.into(), b.into()).simd_into(token)
14028 }
14029 );
14030 kernel(self, a, b)
14031 }
14032 #[inline(always)]
14033 fn saturating_add_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14034 crate::kernel!(
14035 #[inline(always)]
14036 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14037 let a = a.into();
14038 let b = b.into();
14039 let wrapped = _mm256_add_epi64(a, b);
14040 let overflow_bits = _mm256_ternarylogic_epi64::<0x42>(a, b, wrapped);
14041 let overflow_mask = _mm256_srai_epi64::<63>(overflow_bits);
14042 let direction =
14043 _mm256_add_epi64(_mm256_srli_epi64::<63>(a), _mm256_set1_epi64x(i64::MAX));
14044 _mm256_ternarylogic_epi64::<0xca>(overflow_mask, direction, wrapped)
14045 .simd_into(token)
14046 }
14047 );
14048 kernel(self, a, b)
14049 }
14050 #[inline(always)]
14051 fn sub_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14052 crate::kernel!(
14053 #[inline(always)]
14054 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14055 _mm256_sub_epi64(a.into(), b.into()).simd_into(token)
14056 }
14057 );
14058 kernel(self, a, b)
14059 }
14060 #[inline(always)]
14061 fn saturating_sub_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14062 crate::kernel!(
14063 #[inline(always)]
14064 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14065 let a = a.into();
14066 let b = b.into();
14067 let wrapped = _mm256_sub_epi64(a, b);
14068 let overflow_bits = _mm256_ternarylogic_epi64::<0x18>(a, b, wrapped);
14069 let overflow_mask = _mm256_srai_epi64::<63>(overflow_bits);
14070 let direction =
14071 _mm256_add_epi64(_mm256_srli_epi64::<63>(a), _mm256_set1_epi64x(i64::MAX));
14072 _mm256_ternarylogic_epi64::<0xca>(overflow_mask, direction, wrapped)
14073 .simd_into(token)
14074 }
14075 );
14076 kernel(self, a, b)
14077 }
14078 #[inline(always)]
14079 fn mul_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14080 crate::kernel!(
14081 #[inline(always)]
14082 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14083 _mm256_mullo_epi64(a.into(), b.into()).simd_into(token)
14084 }
14085 );
14086 kernel(self, a, b)
14087 }
14088 #[inline(always)]
14089 fn and_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14090 crate::kernel!(
14091 #[inline(always)]
14092 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14093 _mm256_and_si256(a.into(), b.into()).simd_into(token)
14094 }
14095 );
14096 kernel(self, a, b)
14097 }
14098 #[inline(always)]
14099 fn or_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14100 crate::kernel!(
14101 #[inline(always)]
14102 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14103 _mm256_or_si256(a.into(), b.into()).simd_into(token)
14104 }
14105 );
14106 kernel(self, a, b)
14107 }
14108 #[inline(always)]
14109 fn xor_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14110 crate::kernel!(
14111 #[inline(always)]
14112 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14113 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
14114 }
14115 );
14116 kernel(self, a, b)
14117 }
14118 #[inline(always)]
14119 fn not_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
14120 a ^ !0
14121 }
14122 #[inline(always)]
14123 fn shl_i64x4(self, a: i64x4<Self>, shift: u32) -> i64x4<Self> {
14124 crate::kernel!(
14125 #[inline(always)]
14126 fn kernel(token: Avx512, a: i64x4<Avx512>, shift: u32) -> i64x4<Avx512> {
14127 _mm256_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
14128 }
14129 );
14130 kernel(self, a, shift)
14131 }
14132 #[inline(always)]
14133 fn shlv_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14134 crate::kernel!(
14135 #[inline(always)]
14136 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14137 _mm256_sllv_epi64(a.into(), b.into()).simd_into(token)
14138 }
14139 );
14140 kernel(self, a, b)
14141 }
14142 #[inline(always)]
14143 fn shr_i64x4(self, a: i64x4<Self>, shift: u32) -> i64x4<Self> {
14144 crate::kernel!(
14145 #[inline(always)]
14146 fn kernel(token: Avx512, a: i64x4<Avx512>, shift: u32) -> i64x4<Avx512> {
14147 _mm256_sra_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
14148 }
14149 );
14150 kernel(self, a, shift)
14151 }
14152 #[inline(always)]
14153 fn shrv_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14154 crate::kernel!(
14155 #[inline(always)]
14156 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14157 _mm256_srav_epi64(a.into(), b.into()).simd_into(token)
14158 }
14159 );
14160 kernel(self, a, b)
14161 }
14162 #[inline(always)]
14163 fn max_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14164 crate::kernel!(
14165 #[inline(always)]
14166 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14167 _mm256_max_epi64(a.into(), b.into()).simd_into(token)
14168 }
14169 );
14170 kernel(self, a, b)
14171 }
14172 #[inline(always)]
14173 fn min_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14174 crate::kernel!(
14175 #[inline(always)]
14176 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14177 _mm256_min_epi64(a.into(), b.into()).simd_into(token)
14178 }
14179 );
14180 kernel(self, a, b)
14181 }
14182 #[inline(always)]
14183 fn clamp_i64x4(self, a: i64x4<Self>, b: i64x4<Self>, c: i64x4<Self>) -> i64x4<Self> {
14184 debug_assert!(
14185 b.simd_le(c).all_true(),
14186 "clamp requires min <= max for all elements"
14187 );
14188 a.max(b).min(c)
14189 }
14190 #[inline(always)]
14191 fn simd_eq_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> mask64x4<Self> {
14192 crate::kernel!(
14193 #[inline(always)]
14194 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> mask64x4<Avx512> {
14195 mask64x4 {
14196 val: _mm256_cmpeq_epi64_mask(a.into(), b.into()),
14197 simd: token,
14198 }
14199 }
14200 );
14201 kernel(self, a, b)
14202 }
14203 #[inline(always)]
14204 fn simd_ne_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> mask64x4<Self> {
14205 crate::kernel!(
14206 #[inline(always)]
14207 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> mask64x4<Avx512> {
14208 mask64x4 {
14209 val: _mm256_cmpneq_epi64_mask(a.into(), b.into()),
14210 simd: token,
14211 }
14212 }
14213 );
14214 kernel(self, a, b)
14215 }
14216 #[inline(always)]
14217 fn simd_lt_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> mask64x4<Self> {
14218 crate::kernel!(
14219 #[inline(always)]
14220 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> mask64x4<Avx512> {
14221 mask64x4 {
14222 val: _mm256_cmplt_epi64_mask(a.into(), b.into()),
14223 simd: token,
14224 }
14225 }
14226 );
14227 kernel(self, a, b)
14228 }
14229 #[inline(always)]
14230 fn simd_le_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> mask64x4<Self> {
14231 crate::kernel!(
14232 #[inline(always)]
14233 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> mask64x4<Avx512> {
14234 mask64x4 {
14235 val: _mm256_cmple_epi64_mask(a.into(), b.into()),
14236 simd: token,
14237 }
14238 }
14239 );
14240 kernel(self, a, b)
14241 }
14242 #[inline(always)]
14243 fn zip_low_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14244 crate::kernel!(
14245 #[inline(always)]
14246 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14247 _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(0, 4, 1, 5), b.into())
14248 .simd_into(token)
14249 }
14250 );
14251 kernel(self, a, b)
14252 }
14253 #[inline(always)]
14254 fn zip_high_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14255 crate::kernel!(
14256 #[inline(always)]
14257 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14258 _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(2, 6, 3, 7), b.into())
14259 .simd_into(token)
14260 }
14261 );
14262 kernel(self, a, b)
14263 }
14264 #[inline(always)]
14265 fn unzip_low_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14266 crate::kernel!(
14267 #[inline(always)]
14268 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14269 _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(0, 2, 4, 6), b.into())
14270 .simd_into(token)
14271 }
14272 );
14273 kernel(self, a, b)
14274 }
14275 #[inline(always)]
14276 fn unzip_high_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14277 crate::kernel!(
14278 #[inline(always)]
14279 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x4<Avx512> {
14280 _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(1, 3, 5, 7), b.into())
14281 .simd_into(token)
14282 }
14283 );
14284 kernel(self, a, b)
14285 }
14286 #[inline(always)]
14287 fn interleave_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> (i64x4<Self>, i64x4<Self>) {
14288 crate::kernel!(
14289 #[inline(always)]
14290 fn kernel(
14291 token: Avx512,
14292 a: i64x4<Avx512>,
14293 b: i64x4<Avx512>,
14294 ) -> (i64x4<Avx512>, i64x4<Avx512>) {
14295 let a = a.into();
14296 let b = b.into();
14297 (
14298 _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(0, 4, 1, 5), b)
14299 .simd_into(token),
14300 _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(2, 6, 3, 7), b)
14301 .simd_into(token),
14302 )
14303 }
14304 );
14305 kernel(self, a, b)
14306 }
14307 #[inline(always)]
14308 fn deinterleave_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> (i64x4<Self>, i64x4<Self>) {
14309 crate::kernel!(
14310 #[inline(always)]
14311 fn kernel(
14312 token: Avx512,
14313 a: i64x4<Avx512>,
14314 b: i64x4<Avx512>,
14315 ) -> (i64x4<Avx512>, i64x4<Avx512>) {
14316 let a = a.into();
14317 let b = b.into();
14318 (
14319 _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(0, 2, 4, 6), b)
14320 .simd_into(token),
14321 _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(1, 3, 5, 7), b)
14322 .simd_into(token),
14323 )
14324 }
14325 );
14326 kernel(self, a, b)
14327 }
14328 #[inline(always)]
14329 fn select_i64x4(self, a: mask64x4<Self>, b: i64x4<Self>, c: i64x4<Self>) -> i64x4<Self> {
14330 crate::kernel!(
14331 #[inline(always)]
14332 fn kernel(
14333 token: Avx512,
14334 a: mask64x4<Avx512>,
14335 b: i64x4<Avx512>,
14336 c: i64x4<Avx512>,
14337 ) -> i64x4<Avx512> {
14338 _mm256_mask_blend_epi64(a.val, c.into(), b.into()).simd_into(token)
14339 }
14340 );
14341 kernel(self, a, b, c)
14342 }
14343 #[inline(always)]
14344 fn combine_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x8<Self> {
14345 crate::kernel!(
14346 #[inline(always)]
14347 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i64x8<Avx512> {
14348 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
14349 }
14350 );
14351 kernel(self, a, b)
14352 }
14353 #[inline(always)]
14354 fn split_i64x4(self, a: i64x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
14355 crate::kernel!(
14356 #[inline(always)]
14357 fn kernel(token: Avx512, a: i64x4<Avx512>) -> (i64x2<Avx512>, i64x2<Avx512>) {
14358 (
14359 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
14360 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
14361 )
14362 }
14363 );
14364 kernel(self, a)
14365 }
14366 #[inline(always)]
14367 fn neg_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
14368 crate::kernel!(
14369 #[inline(always)]
14370 fn kernel(token: Avx512, a: i64x4<Avx512>) -> i64x4<Avx512> {
14371 _mm256_sub_epi64(_mm256_setzero_si256(), a.into()).simd_into(token)
14372 }
14373 );
14374 kernel(self, a)
14375 }
14376 #[inline(always)]
14377 fn narrow_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i32x8<Self> {
14378 crate::kernel!(
14379 #[inline(always)]
14380 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i32x8<Avx512> {
14381 let low = _mm256_cvtepi64_epi32(a.into());
14382 let high = _mm256_cvtepi64_epi32(b.into());
14383 _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(low), high).simd_into(token)
14384 }
14385 );
14386 kernel(self, a, b)
14387 }
14388 #[inline(always)]
14389 fn saturating_narrow_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i32x8<Self> {
14390 crate::kernel!(
14391 #[inline(always)]
14392 fn kernel(token: Avx512, a: i64x4<Avx512>, b: i64x4<Avx512>) -> i32x8<Avx512> {
14393 let low = _mm256_cvtsepi64_epi32(a.into());
14394 let high = _mm256_cvtsepi64_epi32(b.into());
14395 _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(low), high).simd_into(token)
14396 }
14397 );
14398 kernel(self, a, b)
14399 }
14400 #[inline(always)]
14401 fn relaxed_narrow_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i32x8<Self> {
14402 debug_assert!(
14403 a.as_slice()
14404 .iter()
14405 .chain(b.as_slice())
14406 .all(|&value| { value >= i32::MIN as i64 && value <= i32::MAX as i64 }),
14407 "relaxed_narrow inputs must fit in the destination type",
14408 );
14409 self.narrow_i64x4(a, b)
14410 }
14411 #[inline(always)]
14412 fn cvt_f64_i64x4(self, a: i64x4<Self>) -> f64x4<Self> {
14413 crate::kernel!(
14414 #[inline(always)]
14415 fn kernel(token: Avx512, a: i64x4<Avx512>) -> f64x4<Avx512> {
14416 _mm256_cvtepi64_pd(a.into()).simd_into(token)
14417 }
14418 );
14419 kernel(self, a)
14420 }
14421 #[inline(always)]
14422 fn splat_u64x4(self, val: u64) -> u64x4<Self> {
14423 crate::kernel!(
14424 #[inline(always)]
14425 fn kernel(token: Avx512, val: u64) -> u64x4<Avx512> {
14426 _mm256_set1_epi64x(val.cast_signed()).simd_into(token)
14427 }
14428 );
14429 kernel(self, val)
14430 }
14431 #[inline(always)]
14432 fn reverse_u64x4(self, a: u64x4<Self>) -> u64x4<Self> {
14433 let indices: u8x32<Self> = [
14434 24, 25, 26, 27, 28, 29, 30, 31, 16, 17, 18, 19, 20, 21, 22, 23, 8, 9, 10, 11, 12, 13,
14435 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
14436 ]
14437 .simd_into(self);
14438 self.swizzle_dyn_u64x4(a, indices)
14439 }
14440 #[inline(always)]
14441 fn slide_u64x4<const SHIFT: usize>(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14442 crate::kernel!(
14443 #[inline(always)]
14444 fn kernel(
14445 token: Avx512,
14446 a: u64x4<Avx512>,
14447 b: u64x4<Avx512>,
14448 shift: usize,
14449 ) -> u64x4<Avx512> {
14450 if shift >= 4usize {
14451 return b;
14452 }
14453 let idx = _mm256_add_epi8(
14454 _mm256_setr_epi8(
14455 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
14456 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
14457 ),
14458 _mm256_set1_epi8((shift * 8usize) as i8),
14459 );
14460 let result = _mm256_permutex2var_epi8(
14461 Bytes::to_bytes(a).val.0,
14462 idx,
14463 Bytes::to_bytes(b).val.0,
14464 );
14465 Bytes::from_bytes(u8x32 {
14466 val: crate::support::Aligned256(result),
14467 simd: token,
14468 })
14469 }
14470 );
14471 kernel(self, a, b, SHIFT)
14472 }
14473 #[inline(always)]
14474 fn slide_within_blocks_u64x4<const SHIFT: usize>(
14475 self,
14476 a: u64x4<Self>,
14477 b: u64x4<Self>,
14478 ) -> u64x4<Self> {
14479 if SHIFT == 0 {
14480 return a;
14481 }
14482 if SHIFT >= 2usize {
14483 return b;
14484 }
14485 let a = Bytes::to_bytes(a).val.0;
14486 let b = Bytes::to_bytes(b).val.0;
14487 let result = dyn_alignr_256(self, b, a, SHIFT * 8usize);
14488 Bytes::from_bytes(u8x32 {
14489 val: crate::support::Aligned256(result),
14490 simd: self,
14491 })
14492 }
14493 #[inline(always)]
14494 fn compress_u64x4(self, values: u64x4<Self>, mask: mask64x4<Self>) -> u64x4<Self> {
14495 crate::kernel!(
14496 #[inline(always)]
14497 fn kernel(
14498 token: Avx512,
14499 values: u64x4<Avx512>,
14500 mask: mask64x4<Avx512>,
14501 ) -> u64x4<Avx512> {
14502 _mm256_maskz_compress_epi64(u64::from((mask).val) as u8, values.into())
14503 .simd_into(token)
14504 }
14505 );
14506 kernel(self, values, mask)
14507 }
14508 #[inline(always)]
14509 fn compress_merge_u64x4(
14510 self,
14511 values: u64x4<Self>,
14512 mask: mask64x4<Self>,
14513 merge: u64x4<Self>,
14514 ) -> u64x4<Self> {
14515 crate::kernel!(
14516 #[inline(always)]
14517 fn kernel(
14518 token: Avx512,
14519 values: u64x4<Avx512>,
14520 mask: mask64x4<Avx512>,
14521 merge: u64x4<Avx512>,
14522 ) -> u64x4<Avx512> {
14523 _mm256_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into())
14524 .simd_into(token)
14525 }
14526 );
14527 kernel(self, values, mask, merge)
14528 }
14529 #[inline(always)]
14530 fn expand_u64x4(self, values: u64x4<Self>, mask: mask64x4<Self>) -> u64x4<Self> {
14531 crate::kernel!(
14532 #[inline(always)]
14533 fn kernel(
14534 token: Avx512,
14535 values: u64x4<Avx512>,
14536 mask: mask64x4<Avx512>,
14537 ) -> u64x4<Avx512> {
14538 _mm256_maskz_expand_epi64(u64::from((mask).val) as u8, values.into())
14539 .simd_into(token)
14540 }
14541 );
14542 kernel(self, values, mask)
14543 }
14544 #[inline(always)]
14545 fn expand_merge_u64x4(
14546 self,
14547 values: u64x4<Self>,
14548 mask: mask64x4<Self>,
14549 merge: u64x4<Self>,
14550 ) -> u64x4<Self> {
14551 crate::kernel!(
14552 #[inline(always)]
14553 fn kernel(
14554 token: Avx512,
14555 values: u64x4<Avx512>,
14556 mask: mask64x4<Avx512>,
14557 merge: u64x4<Avx512>,
14558 ) -> u64x4<Avx512> {
14559 _mm256_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into())
14560 .simd_into(token)
14561 }
14562 );
14563 kernel(self, values, mask, merge)
14564 }
14565 #[inline(always)]
14566 fn count_ones_u64x4(self, a: u64x4<Self>) -> u64x4<Self> {
14567 crate::kernel!(
14568 #[inline(always)]
14569 fn kernel(token: Avx512, a: u64x4<Avx512>) -> u64x4<Avx512> {
14570 _mm256_popcnt_epi64(a.into()).simd_into(token)
14571 }
14572 );
14573 kernel(self, a)
14574 }
14575 #[inline(always)]
14576 fn count_zeros_u64x4(self, a: u64x4<Self>) -> u64x4<Self> {
14577 self.count_ones_u64x4(self.not_u64x4(a))
14578 }
14579 #[inline(always)]
14580 fn add_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14581 crate::kernel!(
14582 #[inline(always)]
14583 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14584 _mm256_add_epi64(a.into(), b.into()).simd_into(token)
14585 }
14586 );
14587 kernel(self, a, b)
14588 }
14589 #[inline(always)]
14590 fn saturating_add_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14591 crate::kernel!(
14592 #[inline(always)]
14593 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14594 let a = a.into();
14595 let b = b.into();
14596 let threshold = _mm256_xor_si256(b, _mm256_set1_epi64x(-1));
14597 _mm256_add_epi64(_mm256_min_epu64(a, threshold), b).simd_into(token)
14598 }
14599 );
14600 kernel(self, a, b)
14601 }
14602 #[inline(always)]
14603 fn sub_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14604 crate::kernel!(
14605 #[inline(always)]
14606 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14607 _mm256_sub_epi64(a.into(), b.into()).simd_into(token)
14608 }
14609 );
14610 kernel(self, a, b)
14611 }
14612 #[inline(always)]
14613 fn saturating_sub_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14614 crate::kernel!(
14615 #[inline(always)]
14616 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14617 let a = a.into();
14618 let b = b.into();
14619 _mm256_sub_epi64(_mm256_max_epu64(a, b), b).simd_into(token)
14620 }
14621 );
14622 kernel(self, a, b)
14623 }
14624 #[inline(always)]
14625 fn mul_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14626 crate::kernel!(
14627 #[inline(always)]
14628 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14629 _mm256_mullo_epi64(a.into(), b.into()).simd_into(token)
14630 }
14631 );
14632 kernel(self, a, b)
14633 }
14634 #[inline(always)]
14635 fn and_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14636 crate::kernel!(
14637 #[inline(always)]
14638 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14639 _mm256_and_si256(a.into(), b.into()).simd_into(token)
14640 }
14641 );
14642 kernel(self, a, b)
14643 }
14644 #[inline(always)]
14645 fn or_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14646 crate::kernel!(
14647 #[inline(always)]
14648 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14649 _mm256_or_si256(a.into(), b.into()).simd_into(token)
14650 }
14651 );
14652 kernel(self, a, b)
14653 }
14654 #[inline(always)]
14655 fn xor_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14656 crate::kernel!(
14657 #[inline(always)]
14658 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14659 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
14660 }
14661 );
14662 kernel(self, a, b)
14663 }
14664 #[inline(always)]
14665 fn not_u64x4(self, a: u64x4<Self>) -> u64x4<Self> {
14666 a ^ !0
14667 }
14668 #[inline(always)]
14669 fn shl_u64x4(self, a: u64x4<Self>, shift: u32) -> u64x4<Self> {
14670 crate::kernel!(
14671 #[inline(always)]
14672 fn kernel(token: Avx512, a: u64x4<Avx512>, shift: u32) -> u64x4<Avx512> {
14673 _mm256_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
14674 }
14675 );
14676 kernel(self, a, shift)
14677 }
14678 #[inline(always)]
14679 fn shlv_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14680 crate::kernel!(
14681 #[inline(always)]
14682 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14683 _mm256_sllv_epi64(a.into(), b.into()).simd_into(token)
14684 }
14685 );
14686 kernel(self, a, b)
14687 }
14688 #[inline(always)]
14689 fn shr_u64x4(self, a: u64x4<Self>, shift: u32) -> u64x4<Self> {
14690 crate::kernel!(
14691 #[inline(always)]
14692 fn kernel(token: Avx512, a: u64x4<Avx512>, shift: u32) -> u64x4<Avx512> {
14693 _mm256_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
14694 }
14695 );
14696 kernel(self, a, shift)
14697 }
14698 #[inline(always)]
14699 fn shrv_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14700 crate::kernel!(
14701 #[inline(always)]
14702 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14703 _mm256_srlv_epi64(a.into(), b.into()).simd_into(token)
14704 }
14705 );
14706 kernel(self, a, b)
14707 }
14708 #[inline(always)]
14709 fn max_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14710 crate::kernel!(
14711 #[inline(always)]
14712 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14713 _mm256_max_epu64(a.into(), b.into()).simd_into(token)
14714 }
14715 );
14716 kernel(self, a, b)
14717 }
14718 #[inline(always)]
14719 fn min_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14720 crate::kernel!(
14721 #[inline(always)]
14722 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14723 _mm256_min_epu64(a.into(), b.into()).simd_into(token)
14724 }
14725 );
14726 kernel(self, a, b)
14727 }
14728 #[inline(always)]
14729 fn clamp_u64x4(self, a: u64x4<Self>, b: u64x4<Self>, c: u64x4<Self>) -> u64x4<Self> {
14730 debug_assert!(
14731 b.simd_le(c).all_true(),
14732 "clamp requires min <= max for all elements"
14733 );
14734 a.max(b).min(c)
14735 }
14736 #[inline(always)]
14737 fn simd_eq_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> mask64x4<Self> {
14738 crate::kernel!(
14739 #[inline(always)]
14740 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> mask64x4<Avx512> {
14741 mask64x4 {
14742 val: _mm256_cmpeq_epu64_mask(a.into(), b.into()),
14743 simd: token,
14744 }
14745 }
14746 );
14747 kernel(self, a, b)
14748 }
14749 #[inline(always)]
14750 fn simd_ne_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> mask64x4<Self> {
14751 crate::kernel!(
14752 #[inline(always)]
14753 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> mask64x4<Avx512> {
14754 mask64x4 {
14755 val: _mm256_cmpneq_epu64_mask(a.into(), b.into()),
14756 simd: token,
14757 }
14758 }
14759 );
14760 kernel(self, a, b)
14761 }
14762 #[inline(always)]
14763 fn simd_lt_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> mask64x4<Self> {
14764 crate::kernel!(
14765 #[inline(always)]
14766 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> mask64x4<Avx512> {
14767 mask64x4 {
14768 val: _mm256_cmplt_epu64_mask(a.into(), b.into()),
14769 simd: token,
14770 }
14771 }
14772 );
14773 kernel(self, a, b)
14774 }
14775 #[inline(always)]
14776 fn simd_le_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> mask64x4<Self> {
14777 crate::kernel!(
14778 #[inline(always)]
14779 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> mask64x4<Avx512> {
14780 mask64x4 {
14781 val: _mm256_cmple_epu64_mask(a.into(), b.into()),
14782 simd: token,
14783 }
14784 }
14785 );
14786 kernel(self, a, b)
14787 }
14788 #[inline(always)]
14789 fn zip_low_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14790 crate::kernel!(
14791 #[inline(always)]
14792 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14793 _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(0, 4, 1, 5), b.into())
14794 .simd_into(token)
14795 }
14796 );
14797 kernel(self, a, b)
14798 }
14799 #[inline(always)]
14800 fn zip_high_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14801 crate::kernel!(
14802 #[inline(always)]
14803 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14804 _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(2, 6, 3, 7), b.into())
14805 .simd_into(token)
14806 }
14807 );
14808 kernel(self, a, b)
14809 }
14810 #[inline(always)]
14811 fn unzip_low_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14812 crate::kernel!(
14813 #[inline(always)]
14814 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14815 _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(0, 2, 4, 6), b.into())
14816 .simd_into(token)
14817 }
14818 );
14819 kernel(self, a, b)
14820 }
14821 #[inline(always)]
14822 fn unzip_high_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14823 crate::kernel!(
14824 #[inline(always)]
14825 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x4<Avx512> {
14826 _mm256_permutex2var_epi64(a.into(), _mm256_setr_epi64x(1, 3, 5, 7), b.into())
14827 .simd_into(token)
14828 }
14829 );
14830 kernel(self, a, b)
14831 }
14832 #[inline(always)]
14833 fn interleave_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> (u64x4<Self>, u64x4<Self>) {
14834 crate::kernel!(
14835 #[inline(always)]
14836 fn kernel(
14837 token: Avx512,
14838 a: u64x4<Avx512>,
14839 b: u64x4<Avx512>,
14840 ) -> (u64x4<Avx512>, u64x4<Avx512>) {
14841 let a = a.into();
14842 let b = b.into();
14843 (
14844 _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(0, 4, 1, 5), b)
14845 .simd_into(token),
14846 _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(2, 6, 3, 7), b)
14847 .simd_into(token),
14848 )
14849 }
14850 );
14851 kernel(self, a, b)
14852 }
14853 #[inline(always)]
14854 fn deinterleave_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> (u64x4<Self>, u64x4<Self>) {
14855 crate::kernel!(
14856 #[inline(always)]
14857 fn kernel(
14858 token: Avx512,
14859 a: u64x4<Avx512>,
14860 b: u64x4<Avx512>,
14861 ) -> (u64x4<Avx512>, u64x4<Avx512>) {
14862 let a = a.into();
14863 let b = b.into();
14864 (
14865 _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(0, 2, 4, 6), b)
14866 .simd_into(token),
14867 _mm256_permutex2var_epi64(a, _mm256_setr_epi64x(1, 3, 5, 7), b)
14868 .simd_into(token),
14869 )
14870 }
14871 );
14872 kernel(self, a, b)
14873 }
14874 #[inline(always)]
14875 fn select_u64x4(self, a: mask64x4<Self>, b: u64x4<Self>, c: u64x4<Self>) -> u64x4<Self> {
14876 crate::kernel!(
14877 #[inline(always)]
14878 fn kernel(
14879 token: Avx512,
14880 a: mask64x4<Avx512>,
14881 b: u64x4<Avx512>,
14882 c: u64x4<Avx512>,
14883 ) -> u64x4<Avx512> {
14884 _mm256_mask_blend_epi64(a.val, c.into(), b.into()).simd_into(token)
14885 }
14886 );
14887 kernel(self, a, b, c)
14888 }
14889 #[inline(always)]
14890 fn combine_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x8<Self> {
14891 crate::kernel!(
14892 #[inline(always)]
14893 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u64x8<Avx512> {
14894 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
14895 }
14896 );
14897 kernel(self, a, b)
14898 }
14899 #[inline(always)]
14900 fn split_u64x4(self, a: u64x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
14901 crate::kernel!(
14902 #[inline(always)]
14903 fn kernel(token: Avx512, a: u64x4<Avx512>) -> (u64x2<Avx512>, u64x2<Avx512>) {
14904 (
14905 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
14906 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
14907 )
14908 }
14909 );
14910 kernel(self, a)
14911 }
14912 #[inline(always)]
14913 fn narrow_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u32x8<Self> {
14914 crate::kernel!(
14915 #[inline(always)]
14916 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u32x8<Avx512> {
14917 let low = _mm256_cvtepi64_epi32(a.into());
14918 let high = _mm256_cvtepi64_epi32(b.into());
14919 _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(low), high).simd_into(token)
14920 }
14921 );
14922 kernel(self, a, b)
14923 }
14924 #[inline(always)]
14925 fn saturating_narrow_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u32x8<Self> {
14926 crate::kernel!(
14927 #[inline(always)]
14928 fn kernel(token: Avx512, a: u64x4<Avx512>, b: u64x4<Avx512>) -> u32x8<Avx512> {
14929 let zero = _mm256_setzero_si256();
14930 let ones = _mm256_cmpeq_epi64(zero, zero);
14931 let a = a.into();
14932 let b = b.into();
14933 let a_fits = _mm256_cmpeq_epi64(_mm256_srli_epi64::<32>(a), zero);
14934 let b_fits = _mm256_cmpeq_epi64(_mm256_srli_epi64::<32>(b), zero);
14935 let a = _mm256_or_si256(a, _mm256_xor_si256(a_fits, ones));
14936 let b = _mm256_or_si256(b, _mm256_xor_si256(b_fits, ones));
14937 _mm256_permute4x64_epi64::<0xd8>(_mm256_castps_si256(_mm256_shuffle_ps::<0x88>(
14938 _mm256_castsi256_ps(a),
14939 _mm256_castsi256_ps(b),
14940 )))
14941 .simd_into(token)
14942 }
14943 );
14944 kernel(self, a, b)
14945 }
14946 #[inline(always)]
14947 fn relaxed_narrow_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u32x8<Self> {
14948 debug_assert!(
14949 a.as_slice()
14950 .iter()
14951 .chain(b.as_slice())
14952 .all(|&value| { value >= u32::MIN as u64 && value <= u32::MAX as u64 }),
14953 "relaxed_narrow inputs must fit in the destination type",
14954 );
14955 self.narrow_u64x4(a, b)
14956 }
14957 #[inline(always)]
14958 fn cvt_f64_u64x4(self, a: u64x4<Self>) -> f64x4<Self> {
14959 crate::kernel!(
14960 #[inline(always)]
14961 fn kernel(token: Avx512, a: u64x4<Avx512>) -> f64x4<Avx512> {
14962 _mm256_cvtepu64_pd(a.into()).simd_into(token)
14963 }
14964 );
14965 kernel(self, a)
14966 }
14967 #[inline(always)]
14968 fn splat_mask64x4(self, val: bool) -> mask64x4<Self> {
14969 mask64x4 {
14970 val: (if val { 15u64 } else { 0 }) as _,
14971 simd: self,
14972 }
14973 }
14974 #[inline(always)]
14975 fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4<Self> {
14976 mask64x4 {
14977 val: (bits & 15u64) as _,
14978 simd: self,
14979 }
14980 }
14981 #[inline(always)]
14982 fn to_bitmask_mask64x4(self, a: mask64x4<Self>) -> u64 {
14983 u64::from((a).val) & 15u64
14984 }
14985 #[inline(always)]
14986 fn from_vector_mask64x4(self, vector: i64x4<Self>) -> mask64x4<Self> {
14987 crate::kernel!(
14988 #[inline(always)]
14989 fn kernel(token: Avx512, vector: i64x4<Avx512>) -> mask64x4<Avx512> {
14990 _mm256_movepi64_mask(vector.into()).simd_into(token)
14991 }
14992 );
14993 kernel(self, vector)
14994 }
14995 #[inline(always)]
14996 fn to_vector_mask64x4(self, a: mask64x4<Self>) -> i64x4<Self> {
14997 crate::kernel!(
14998 #[inline(always)]
14999 fn kernel(token: Avx512, a: mask64x4<Avx512>) -> i64x4<Avx512> {
15000 _mm256_movm_epi64(a.val).simd_into(token)
15001 }
15002 );
15003 kernel(self, a)
15004 }
15005 #[inline(always)]
15006 fn set_mask64x4(self, a: &mut mask64x4<Self>, index: usize, value: bool) -> () {
15007 assert!(
15008 index < 4usize,
15009 "mask lane index {index} is out of bounds for {} lanes",
15010 4usize
15011 );
15012 let bit = 1u64 << index;
15013 let bits = u64::from((a).val);
15014 let bits = if value { bits | bit } else { bits & !bit };
15015 *a = mask64x4 {
15016 val: (bits) as _,
15017 simd: self,
15018 };
15019 }
15020 #[inline(always)]
15021 fn rotate_elements_left_mask64x4<const OFFSET: usize>(
15022 self,
15023 a: mask64x4<Self>,
15024 ) -> mask64x4<Self> {
15025 let bits = u64::from((a).val) & 15u64;
15026 let offset = OFFSET % 4;
15027 let bits = if offset == 0 {
15028 bits
15029 } else {
15030 ((bits >> offset) | (bits << (4 - offset))) & 15u64
15031 };
15032 mask64x4 {
15033 val: (bits) as _,
15034 simd: self,
15035 }
15036 }
15037 #[inline(always)]
15038 fn rotate_elements_right_mask64x4<const OFFSET: usize>(
15039 self,
15040 a: mask64x4<Self>,
15041 ) -> mask64x4<Self> {
15042 let bits = u64::from((a).val) & 15u64;
15043 let offset = OFFSET % 4;
15044 let bits = if offset == 0 {
15045 bits
15046 } else {
15047 ((bits << offset) | (bits >> (4 - offset))) & 15u64
15048 };
15049 mask64x4 {
15050 val: (bits) as _,
15051 simd: self,
15052 }
15053 }
15054 #[inline(always)]
15055 fn and_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
15056 mask64x4 {
15057 val: ((u64::from((a).val) & u64::from((b).val)) & 15u64) as _,
15058 simd: self,
15059 }
15060 }
15061 #[inline(always)]
15062 fn or_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
15063 mask64x4 {
15064 val: ((u64::from((a).val) | u64::from((b).val)) & 15u64) as _,
15065 simd: self,
15066 }
15067 }
15068 #[inline(always)]
15069 fn xor_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
15070 mask64x4 {
15071 val: ((u64::from((a).val) ^ u64::from((b).val)) & 15u64) as _,
15072 simd: self,
15073 }
15074 }
15075 #[inline(always)]
15076 fn not_mask64x4(self, a: mask64x4<Self>) -> mask64x4<Self> {
15077 mask64x4 {
15078 val: ((!u64::from((a).val)) & 15u64) as _,
15079 simd: self,
15080 }
15081 }
15082 #[inline(always)]
15083 fn reverse_mask64x4(self, a: mask64x4<Self>) -> mask64x4<Self> {
15084 mask64x4 {
15085 val: (a.val.reverse_bits() >> 4usize) as _,
15086 simd: self,
15087 }
15088 }
15089 #[inline(always)]
15090 fn select_mask64x4(
15091 self,
15092 a: mask64x4<Self>,
15093 b: mask64x4<Self>,
15094 c: mask64x4<Self>,
15095 ) -> mask64x4<Self> {
15096 mask64x4 {
15097 val: (((u64::from((a).val) & u64::from((b).val))
15098 | ((!u64::from((a).val)) & u64::from((c).val)))
15099 & 15u64) as _,
15100 simd: self,
15101 }
15102 }
15103 #[inline(always)]
15104 fn simd_eq_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
15105 mask64x4 {
15106 val: (!u64::from(a.val ^ b.val) & 15u64) as _,
15107 simd: self,
15108 }
15109 }
15110 #[inline(always)]
15111 fn simd_ne_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
15112 mask64x4 {
15113 val: (u64::from(a.val ^ b.val) & 15u64) as _,
15114 simd: self,
15115 }
15116 }
15117 #[inline(always)]
15118 fn any_true_mask64x4(self, a: mask64x4<Self>) -> bool {
15119 let bits = u64::from((a).val) & 15u64;
15120 bits != 0
15121 }
15122 #[inline(always)]
15123 fn all_true_mask64x4(self, a: mask64x4<Self>) -> bool {
15124 let bits = u64::from((a).val) & 15u64;
15125 bits == 15u64
15126 }
15127 #[inline(always)]
15128 fn any_false_mask64x4(self, a: mask64x4<Self>) -> bool {
15129 let bits = u64::from((a).val) & 15u64;
15130 bits != 15u64
15131 }
15132 #[inline(always)]
15133 fn all_false_mask64x4(self, a: mask64x4<Self>) -> bool {
15134 let bits = u64::from((a).val) & 15u64;
15135 bits == 0
15136 }
15137 #[inline(always)]
15138 fn combine_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x8<Self> {
15139 let bits = (u64::from(a.val) | (u64::from(b.val) << 4usize)) & 255u64;
15140 mask64x8 {
15141 val: bits as _,
15142 simd: self,
15143 }
15144 }
15145 #[inline(always)]
15146 fn split_mask64x4(self, a: mask64x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
15147 let bits = u64::from(a.val);
15148 (
15149 mask64x2 {
15150 val: (bits & 3u64) as _,
15151 simd: self,
15152 },
15153 mask64x2 {
15154 val: ((bits >> 2usize) & 3u64) as _,
15155 simd: self,
15156 },
15157 )
15158 }
15159 #[inline(always)]
15160 fn narrow_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask32x8<Self> {
15161 let low = u64::from(a.val) & 15u64;
15162 let high = u64::from(b.val) & 15u64;
15163 mask32x8 {
15164 val: (low | (high << 4usize)) as _,
15165 simd: self,
15166 }
15167 }
15168 #[inline(always)]
15169 fn abs_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
15170 crate::kernel!(
15171 #[inline(always)]
15172 fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
15173 _mm512_andnot_ps(_mm512_set1_ps(-0.0), a.into()).simd_into(token)
15174 }
15175 );
15176 kernel(self, a)
15177 }
15178 #[inline(always)]
15179 fn splat_f32x16(self, val: f32) -> f32x16<Self> {
15180 crate::kernel!(
15181 #[inline(always)]
15182 fn kernel(token: Avx512, val: f32) -> f32x16<Avx512> {
15183 _mm512_set1_ps(val).simd_into(token)
15184 }
15185 );
15186 kernel(self, val)
15187 }
15188 #[inline(always)]
15189 fn reverse_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
15190 let indices: u8x64<Self> = [
15191 60, 61, 62, 63, 56, 57, 58, 59, 52, 53, 54, 55, 48, 49, 50, 51, 44, 45, 46, 47, 40, 41,
15192 42, 43, 36, 37, 38, 39, 32, 33, 34, 35, 28, 29, 30, 31, 24, 25, 26, 27, 20, 21, 22, 23,
15193 16, 17, 18, 19, 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3,
15194 ]
15195 .simd_into(self);
15196 self.swizzle_dyn_f32x16(a, indices)
15197 }
15198 #[inline(always)]
15199 fn slide_f32x16<const SHIFT: usize>(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15200 crate::kernel!(
15201 #[inline(always)]
15202 fn kernel(
15203 token: Avx512,
15204 a: f32x16<Avx512>,
15205 b: f32x16<Avx512>,
15206 shift: usize,
15207 ) -> f32x16<Avx512> {
15208 if shift >= 16usize {
15209 return b;
15210 }
15211 let idx = _mm512_add_epi8(
15212 _mm512_set_epi8(
15213 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
15214 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
15215 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
15216 5, 4, 3, 2, 1, 0,
15217 ),
15218 _mm512_set1_epi8((shift * 4usize) as i8),
15219 );
15220 let result = _mm512_permutex2var_epi8(
15221 Bytes::to_bytes(a).val.0,
15222 idx,
15223 Bytes::to_bytes(b).val.0,
15224 );
15225 Bytes::from_bytes(u8x64 {
15226 val: crate::support::Aligned512(result),
15227 simd: token,
15228 })
15229 }
15230 );
15231 kernel(self, a, b, SHIFT)
15232 }
15233 #[inline(always)]
15234 fn slide_within_blocks_f32x16<const SHIFT: usize>(
15235 self,
15236 a: f32x16<Self>,
15237 b: f32x16<Self>,
15238 ) -> f32x16<Self> {
15239 if SHIFT == 0 {
15240 return a;
15241 }
15242 if SHIFT >= 4usize {
15243 return b;
15244 }
15245 let a = Bytes::to_bytes(a).val.0;
15246 let b = Bytes::to_bytes(b).val.0;
15247 let result = dyn_alignr_512(self, b, a, SHIFT * 4usize);
15248 Bytes::from_bytes(u8x64 {
15249 val: crate::support::Aligned512(result),
15250 simd: self,
15251 })
15252 }
15253 #[inline(always)]
15254 fn compress_f32x16(self, values: f32x16<Self>, mask: mask32x16<Self>) -> f32x16<Self> {
15255 crate::kernel!(
15256 #[inline(always)]
15257 fn kernel(
15258 token: Avx512,
15259 values: f32x16<Avx512>,
15260 mask: mask32x16<Avx512>,
15261 ) -> f32x16<Avx512> {
15262 _mm512_maskz_compress_ps(u64::from((mask).val) as u16, values.into())
15263 .simd_into(token)
15264 }
15265 );
15266 kernel(self, values, mask)
15267 }
15268 #[inline(always)]
15269 fn compress_merge_f32x16(
15270 self,
15271 values: f32x16<Self>,
15272 mask: mask32x16<Self>,
15273 merge: f32x16<Self>,
15274 ) -> f32x16<Self> {
15275 crate::kernel!(
15276 #[inline(always)]
15277 fn kernel(
15278 token: Avx512,
15279 values: f32x16<Avx512>,
15280 mask: mask32x16<Avx512>,
15281 merge: f32x16<Avx512>,
15282 ) -> f32x16<Avx512> {
15283 _mm512_mask_compress_ps(merge.into(), u64::from((mask).val) as u16, values.into())
15284 .simd_into(token)
15285 }
15286 );
15287 kernel(self, values, mask, merge)
15288 }
15289 #[inline(always)]
15290 fn expand_f32x16(self, values: f32x16<Self>, mask: mask32x16<Self>) -> f32x16<Self> {
15291 crate::kernel!(
15292 #[inline(always)]
15293 fn kernel(
15294 token: Avx512,
15295 values: f32x16<Avx512>,
15296 mask: mask32x16<Avx512>,
15297 ) -> f32x16<Avx512> {
15298 _mm512_maskz_expand_ps(u64::from((mask).val) as u16, values.into()).simd_into(token)
15299 }
15300 );
15301 kernel(self, values, mask)
15302 }
15303 #[inline(always)]
15304 fn expand_merge_f32x16(
15305 self,
15306 values: f32x16<Self>,
15307 mask: mask32x16<Self>,
15308 merge: f32x16<Self>,
15309 ) -> f32x16<Self> {
15310 crate::kernel!(
15311 #[inline(always)]
15312 fn kernel(
15313 token: Avx512,
15314 values: f32x16<Avx512>,
15315 mask: mask32x16<Avx512>,
15316 merge: f32x16<Avx512>,
15317 ) -> f32x16<Avx512> {
15318 _mm512_mask_expand_ps(merge.into(), u64::from((mask).val) as u16, values.into())
15319 .simd_into(token)
15320 }
15321 );
15322 kernel(self, values, mask, merge)
15323 }
15324 #[inline(always)]
15325 fn neg_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
15326 crate::kernel!(
15327 #[inline(always)]
15328 fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
15329 _mm512_xor_ps(a.into(), _mm512_set1_ps(-0.0)).simd_into(token)
15330 }
15331 );
15332 kernel(self, a)
15333 }
15334 #[inline(always)]
15335 fn sqrt_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
15336 crate::kernel!(
15337 #[inline(always)]
15338 fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
15339 _mm512_sqrt_ps(a.into()).simd_into(token)
15340 }
15341 );
15342 kernel(self, a)
15343 }
15344 #[inline(always)]
15345 fn approximate_recip_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
15346 crate::kernel!(
15347 #[inline(always)]
15348 fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
15349 _mm512_rcp14_ps(a.into()).simd_into(token)
15350 }
15351 );
15352 kernel(self, a)
15353 }
15354 #[inline(always)]
15355 fn add_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15356 crate::kernel!(
15357 #[inline(always)]
15358 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
15359 _mm512_add_ps(a.into(), b.into()).simd_into(token)
15360 }
15361 );
15362 kernel(self, a, b)
15363 }
15364 #[inline(always)]
15365 fn sub_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15366 crate::kernel!(
15367 #[inline(always)]
15368 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
15369 _mm512_sub_ps(a.into(), b.into()).simd_into(token)
15370 }
15371 );
15372 kernel(self, a, b)
15373 }
15374 #[inline(always)]
15375 fn mul_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15376 crate::kernel!(
15377 #[inline(always)]
15378 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
15379 _mm512_mul_ps(a.into(), b.into()).simd_into(token)
15380 }
15381 );
15382 kernel(self, a, b)
15383 }
15384 #[inline(always)]
15385 fn div_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15386 crate::kernel!(
15387 #[inline(always)]
15388 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
15389 _mm512_div_ps(a.into(), b.into()).simd_into(token)
15390 }
15391 );
15392 kernel(self, a, b)
15393 }
15394 #[inline(always)]
15395 fn copysign_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15396 crate::kernel!(
15397 #[inline(always)]
15398 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
15399 let mask = _mm512_set1_ps(-0.0);
15400 _mm512_or_ps(
15401 _mm512_and_ps(mask, b.into()),
15402 _mm512_andnot_ps(mask, a.into()),
15403 )
15404 .simd_into(token)
15405 }
15406 );
15407 kernel(self, a, b)
15408 }
15409 #[inline(always)]
15410 fn max_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15411 crate::kernel!(
15412 #[inline(always)]
15413 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
15414 _mm512_max_ps(a.into(), b.into()).simd_into(token)
15415 }
15416 );
15417 kernel(self, a, b)
15418 }
15419 #[inline(always)]
15420 fn min_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15421 crate::kernel!(
15422 #[inline(always)]
15423 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
15424 _mm512_min_ps(a.into(), b.into()).simd_into(token)
15425 }
15426 );
15427 kernel(self, a, b)
15428 }
15429 #[inline(always)]
15430 fn max_precise_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15431 crate::kernel!(
15432 #[inline(always)]
15433 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
15434 _mm512_range_ps::<5i32>(a.into(), b.into()).simd_into(token)
15435 }
15436 );
15437 kernel(self, a, b)
15438 }
15439 #[inline(always)]
15440 fn min_precise_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15441 crate::kernel!(
15442 #[inline(always)]
15443 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
15444 _mm512_range_ps::<4i32>(a.into(), b.into()).simd_into(token)
15445 }
15446 );
15447 kernel(self, a, b)
15448 }
15449 #[inline(always)]
15450 fn clamp_f32x16(self, a: f32x16<Self>, b: f32x16<Self>, c: f32x16<Self>) -> f32x16<Self> {
15451 crate::kernel!(
15452 #[inline(always)]
15453 fn kernel(
15454 token: Avx512,
15455 a: f32x16<Avx512>,
15456 b: f32x16<Avx512>,
15457 c: f32x16<Avx512>,
15458 ) -> f32x16<Avx512> {
15459 debug_assert!(
15460 b.simd_le(c).all_true(),
15461 "clamp requires min <= max for all elements"
15462 );
15463 _mm512_min_ps(c.into(), _mm512_max_ps(b.into(), a.into())).simd_into(token)
15464 }
15465 );
15466 kernel(self, a, b, c)
15467 }
15468 #[inline(always)]
15469 fn simd_eq_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
15470 crate::kernel!(
15471 #[inline(always)]
15472 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> mask32x16<Avx512> {
15473 mask32x16 {
15474 val: _mm512_cmp_ps_mask::<0i32>(a.into(), b.into()),
15475 simd: token,
15476 }
15477 }
15478 );
15479 kernel(self, a, b)
15480 }
15481 #[inline(always)]
15482 fn simd_ne_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
15483 crate::kernel!(
15484 #[inline(always)]
15485 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> mask32x16<Avx512> {
15486 mask32x16 {
15487 val: _mm512_cmp_ps_mask::<4i32>(a.into(), b.into()),
15488 simd: token,
15489 }
15490 }
15491 );
15492 kernel(self, a, b)
15493 }
15494 #[inline(always)]
15495 fn simd_lt_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
15496 crate::kernel!(
15497 #[inline(always)]
15498 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> mask32x16<Avx512> {
15499 mask32x16 {
15500 val: _mm512_cmp_ps_mask::<17i32>(a.into(), b.into()),
15501 simd: token,
15502 }
15503 }
15504 );
15505 kernel(self, a, b)
15506 }
15507 #[inline(always)]
15508 fn simd_le_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
15509 crate::kernel!(
15510 #[inline(always)]
15511 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> mask32x16<Avx512> {
15512 mask32x16 {
15513 val: _mm512_cmp_ps_mask::<18i32>(a.into(), b.into()),
15514 simd: token,
15515 }
15516 }
15517 );
15518 kernel(self, a, b)
15519 }
15520 #[inline(always)]
15521 fn zip_low_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15522 crate::kernel!(
15523 #[inline(always)]
15524 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
15525 _mm512_permutex2var_ps(
15526 a.into(),
15527 _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
15528 b.into(),
15529 )
15530 .simd_into(token)
15531 }
15532 );
15533 kernel(self, a, b)
15534 }
15535 #[inline(always)]
15536 fn zip_high_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15537 crate::kernel!(
15538 #[inline(always)]
15539 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
15540 _mm512_permutex2var_ps(
15541 a.into(),
15542 _mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31),
15543 b.into(),
15544 )
15545 .simd_into(token)
15546 }
15547 );
15548 kernel(self, a, b)
15549 }
15550 #[inline(always)]
15551 fn unzip_low_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15552 crate::kernel!(
15553 #[inline(always)]
15554 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
15555 _mm512_permutex2var_ps(
15556 a.into(),
15557 _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
15558 b.into(),
15559 )
15560 .simd_into(token)
15561 }
15562 );
15563 kernel(self, a, b)
15564 }
15565 #[inline(always)]
15566 fn unzip_high_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15567 crate::kernel!(
15568 #[inline(always)]
15569 fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
15570 _mm512_permutex2var_ps(
15571 a.into(),
15572 _mm512_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
15573 b.into(),
15574 )
15575 .simd_into(token)
15576 }
15577 );
15578 kernel(self, a, b)
15579 }
15580 #[inline(always)]
15581 fn interleave_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> (f32x16<Self>, f32x16<Self>) {
15582 crate::kernel!(
15583 #[inline(always)]
15584 fn kernel(
15585 token: Avx512,
15586 a: f32x16<Avx512>,
15587 b: f32x16<Avx512>,
15588 ) -> (f32x16<Avx512>, f32x16<Avx512>) {
15589 let a = a.into();
15590 let b = b.into();
15591 (
15592 _mm512_permutex2var_ps(
15593 a,
15594 _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
15595 b,
15596 )
15597 .simd_into(token),
15598 _mm512_permutex2var_ps(
15599 a,
15600 _mm512_setr_epi32(
15601 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31,
15602 ),
15603 b,
15604 )
15605 .simd_into(token),
15606 )
15607 }
15608 );
15609 kernel(self, a, b)
15610 }
15611 #[inline(always)]
15612 fn deinterleave_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> (f32x16<Self>, f32x16<Self>) {
15613 crate::kernel!(
15614 #[inline(always)]
15615 fn kernel(
15616 token: Avx512,
15617 a: f32x16<Avx512>,
15618 b: f32x16<Avx512>,
15619 ) -> (f32x16<Avx512>, f32x16<Avx512>) {
15620 let a = a.into();
15621 let b = b.into();
15622 (
15623 _mm512_permutex2var_ps(
15624 a,
15625 _mm512_setr_epi32(
15626 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
15627 ),
15628 b,
15629 )
15630 .simd_into(token),
15631 _mm512_permutex2var_ps(
15632 a,
15633 _mm512_setr_epi32(
15634 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31,
15635 ),
15636 b,
15637 )
15638 .simd_into(token),
15639 )
15640 }
15641 );
15642 kernel(self, a, b)
15643 }
15644 #[inline(always)]
15645 fn mul_add_f32x16(self, a: f32x16<Self>, b: f32x16<Self>, c: f32x16<Self>) -> f32x16<Self> {
15646 crate::kernel!(
15647 #[inline(always)]
15648 fn kernel(
15649 token: Avx512,
15650 a: f32x16<Avx512>,
15651 b: f32x16<Avx512>,
15652 c: f32x16<Avx512>,
15653 ) -> f32x16<Avx512> {
15654 _mm512_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token)
15655 }
15656 );
15657 kernel(self, a, b, c)
15658 }
15659 #[inline(always)]
15660 fn mul_add_precise_f32x16(
15661 self,
15662 a: f32x16<Self>,
15663 b: f32x16<Self>,
15664 c: f32x16<Self>,
15665 ) -> f32x16<Self> {
15666 self.mul_add_f32x16(a, b, c)
15667 }
15668 #[inline(always)]
15669 fn mul_sub_f32x16(self, a: f32x16<Self>, b: f32x16<Self>, c: f32x16<Self>) -> f32x16<Self> {
15670 crate::kernel!(
15671 #[inline(always)]
15672 fn kernel(
15673 token: Avx512,
15674 a: f32x16<Avx512>,
15675 b: f32x16<Avx512>,
15676 c: f32x16<Avx512>,
15677 ) -> f32x16<Avx512> {
15678 _mm512_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token)
15679 }
15680 );
15681 kernel(self, a, b, c)
15682 }
15683 #[inline(always)]
15684 fn mul_sub_precise_f32x16(
15685 self,
15686 a: f32x16<Self>,
15687 b: f32x16<Self>,
15688 c: f32x16<Self>,
15689 ) -> f32x16<Self> {
15690 self.mul_sub_f32x16(a, b, c)
15691 }
15692 #[inline(always)]
15693 fn floor_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
15694 crate::kernel!(
15695 #[inline(always)]
15696 fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
15697 _mm512_roundscale_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
15698 .simd_into(token)
15699 }
15700 );
15701 kernel(self, a)
15702 }
15703 #[inline(always)]
15704 fn ceil_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
15705 crate::kernel!(
15706 #[inline(always)]
15707 fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
15708 _mm512_roundscale_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
15709 .simd_into(token)
15710 }
15711 );
15712 kernel(self, a)
15713 }
15714 #[inline(always)]
15715 fn round_ties_even_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
15716 crate::kernel!(
15717 #[inline(always)]
15718 fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
15719 _mm512_roundscale_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
15720 .simd_into(token)
15721 }
15722 );
15723 kernel(self, a)
15724 }
15725 #[inline(always)]
15726 fn round_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
15727 let bias = self.splat_f32x16(const { f32::next_down(0.5) });
15728 (a + self.copysign_f32x16(bias, a)).trunc()
15729 }
15730 #[inline(always)]
15731 fn fract_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
15732 a - self.trunc_f32x16(a)
15733 }
15734 #[inline(always)]
15735 fn trunc_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
15736 crate::kernel!(
15737 #[inline(always)]
15738 fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
15739 _mm512_roundscale_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
15740 .simd_into(token)
15741 }
15742 );
15743 kernel(self, a)
15744 }
15745 #[inline(always)]
15746 fn to_degrees_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
15747 a * const { f32::to_degrees(1.0) }
15748 }
15749 #[inline(always)]
15750 fn to_radians_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
15751 a * const { f32::to_radians(1.0) }
15752 }
15753 #[inline(always)]
15754 fn is_nan_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15755 crate::kernel!(
15756 #[inline(always)]
15757 fn kernel(token: Avx512, a: f32x16<Avx512>) -> mask32x16<Avx512> {
15758 mask32x16 {
15759 val: _mm512_fpclass_ps_mask::<129i32>(a.into()),
15760 simd: token,
15761 }
15762 }
15763 );
15764 kernel(self, a)
15765 }
15766 #[inline(always)]
15767 fn is_infinite_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15768 crate::kernel!(
15769 #[inline(always)]
15770 fn kernel(token: Avx512, a: f32x16<Avx512>) -> mask32x16<Avx512> {
15771 mask32x16 {
15772 val: _mm512_fpclass_ps_mask::<24i32>(a.into()),
15773 simd: token,
15774 }
15775 }
15776 );
15777 kernel(self, a)
15778 }
15779 #[inline(always)]
15780 fn is_finite_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15781 crate::kernel!(
15782 #[inline(always)]
15783 fn kernel(token: Avx512, a: f32x16<Avx512>) -> mask32x16<Avx512> {
15784 !(mask32x16 {
15785 val: _mm512_fpclass_ps_mask::<153i32>(a.into()),
15786 simd: token,
15787 })
15788 }
15789 );
15790 kernel(self, a)
15791 }
15792 #[inline(always)]
15793 fn is_subnormal_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15794 crate::kernel!(
15795 #[inline(always)]
15796 fn kernel(token: Avx512, a: f32x16<Avx512>) -> mask32x16<Avx512> {
15797 mask32x16 {
15798 val: _mm512_fpclass_ps_mask::<129i32>(_mm512_xor_ps(
15799 a.into(),
15800 _mm512_set1_ps(f32::INFINITY),
15801 )),
15802 simd: token,
15803 }
15804 }
15805 );
15806 kernel(self, a)
15807 }
15808 #[inline(always)]
15809 fn is_normal_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15810 crate::kernel!(
15811 #[inline(always)]
15812 fn kernel(token: Avx512, a: f32x16<Avx512>) -> mask32x16<Avx512> {
15813 !(mask32x16 {
15814 val: _mm512_fpclass_ps_mask::<191i32>(a.into()),
15815 simd: token,
15816 })
15817 }
15818 );
15819 kernel(self, a)
15820 }
15821 #[inline(always)]
15822 fn is_sign_positive_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15823 let i: i32x16<Self> = a.bitcast();
15824 (i & i32::MIN).simd_eq(0)
15825 }
15826 #[inline(always)]
15827 fn is_sign_negative_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15828 !a.is_sign_positive()
15829 }
15830 #[inline(always)]
15831 fn select_f32x16(self, a: mask32x16<Self>, b: f32x16<Self>, c: f32x16<Self>) -> f32x16<Self> {
15832 crate::kernel!(
15833 #[inline(always)]
15834 fn kernel(
15835 token: Avx512,
15836 a: mask32x16<Avx512>,
15837 b: f32x16<Avx512>,
15838 c: f32x16<Avx512>,
15839 ) -> f32x16<Avx512> {
15840 _mm512_mask_blend_ps(a.val, c.into(), b.into()).simd_into(token)
15841 }
15842 );
15843 kernel(self, a, b, c)
15844 }
15845 #[inline(always)]
15846 fn split_f32x16(self, a: f32x16<Self>) -> (f32x8<Self>, f32x8<Self>) {
15847 crate::kernel!(
15848 #[inline(always)]
15849 fn kernel(token: Avx512, a: f32x16<Avx512>) -> (f32x8<Avx512>, f32x8<Avx512>) {
15850 (
15851 _mm512_castps512_ps256(a.into()).simd_into(token),
15852 _mm512_extractf32x8_ps::<1>(a.into()).simd_into(token),
15853 )
15854 }
15855 );
15856 kernel(self, a)
15857 }
15858 #[inline(always)]
15859 fn widen_f32x16(self, a: f32x16<Self>) -> (f64x8<Self>, f64x8<Self>) {
15860 crate::kernel!(
15861 #[inline(always)]
15862 fn kernel(token: Avx512, a: f32x16<Avx512>) -> (f64x8<Avx512>, f64x8<Avx512>) {
15863 let raw = a.into();
15864 (
15865 _mm512_cvtps_pd(_mm512_castps512_ps256(raw)).simd_into(token),
15866 _mm512_cvtps_pd(_mm512_extractf32x8_ps::<1>(raw)).simd_into(token),
15867 )
15868 }
15869 );
15870 kernel(self, a)
15871 }
15872 #[inline(always)]
15873 fn cvt_u32_f32x16(self, a: f32x16<Self>) -> u32x16<Self> {
15874 crate::kernel!(
15875 #[inline(always)]
15876 fn kernel(token: Avx512, a: f32x16<Avx512>) -> u32x16<Avx512> {
15877 _mm512_cvttps_epu32(a.into()).simd_into(token)
15878 }
15879 );
15880 kernel(self, a)
15881 }
15882 #[inline(always)]
15883 fn cvt_u32_precise_f32x16(self, a: f32x16<Self>) -> u32x16<Self> {
15884 crate::kernel!(
15885 #[inline(always)]
15886 fn kernel(token: Avx512, a: f32x16<Avx512>) -> u32x16<Avx512> {
15887 let a = a.into();
15888 let positive = _mm512_cmp_ps_mask::<17i32>(_mm512_setzero_ps(), a);
15889 _mm512_maskz_cvttps_epu32(positive, a).simd_into(token)
15890 }
15891 );
15892 kernel(self, a)
15893 }
15894 #[inline(always)]
15895 fn cvt_i32_f32x16(self, a: f32x16<Self>) -> i32x16<Self> {
15896 crate::kernel!(
15897 #[inline(always)]
15898 fn kernel(token: Avx512, a: f32x16<Avx512>) -> i32x16<Avx512> {
15899 _mm512_cvttps_epi32(a.into()).simd_into(token)
15900 }
15901 );
15902 kernel(self, a)
15903 }
15904 #[inline(always)]
15905 fn cvt_i32_precise_f32x16(self, a: f32x16<Self>) -> i32x16<Self> {
15906 crate::kernel!(
15907 #[inline(always)]
15908 fn kernel(token: Avx512, a: f32x16<Avx512>) -> i32x16<Avx512> {
15909 let a = a.into();
15910 let in_range = _mm512_cmp_ps_mask::<17i32>(a, _mm512_set1_ps(2147483648.0));
15911 let mut converted =
15912 _mm512_mask_cvttps_epi32(_mm512_set1_epi32(i32::MAX), in_range, a);
15913 let is_not_nan = _mm512_cmp_ps_mask::<7i32>(a, a);
15914 converted = _mm512_mask_blend_epi32(is_not_nan, _mm512_setzero_si512(), converted);
15915 converted.simd_into(token)
15916 }
15917 );
15918 kernel(self, a)
15919 }
15920 #[inline(always)]
15921 fn abs_i8x64(self, a: i8x64<Self>) -> i8x64<Self> {
15922 crate::kernel!(
15923 #[inline(always)]
15924 fn kernel(token: Avx512, a: i8x64<Avx512>) -> i8x64<Avx512> {
15925 _mm512_abs_epi8(a.into()).simd_into(token)
15926 }
15927 );
15928 kernel(self, a)
15929 }
15930 #[inline(always)]
15931 fn splat_i8x64(self, val: i8) -> i8x64<Self> {
15932 crate::kernel!(
15933 #[inline(always)]
15934 fn kernel(token: Avx512, val: i8) -> i8x64<Avx512> {
15935 _mm512_set1_epi8(val).simd_into(token)
15936 }
15937 );
15938 kernel(self, val)
15939 }
15940 #[inline(always)]
15941 fn reverse_i8x64(self, a: i8x64<Self>) -> i8x64<Self> {
15942 let indices: u8x64<Self> = [
15943 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42,
15944 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20,
15945 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
15946 ]
15947 .simd_into(self);
15948 self.swizzle_dyn_i8x64(a, indices)
15949 }
15950 #[inline(always)]
15951 fn slide_i8x64<const SHIFT: usize>(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
15952 crate::kernel!(
15953 #[inline(always)]
15954 fn kernel(
15955 token: Avx512,
15956 a: i8x64<Avx512>,
15957 b: i8x64<Avx512>,
15958 shift: usize,
15959 ) -> i8x64<Avx512> {
15960 if shift >= 64usize {
15961 return b;
15962 }
15963 let idx = _mm512_add_epi8(
15964 _mm512_set_epi8(
15965 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
15966 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
15967 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
15968 5, 4, 3, 2, 1, 0,
15969 ),
15970 _mm512_set1_epi8((shift) as i8),
15971 );
15972 let result = _mm512_permutex2var_epi8(
15973 Bytes::to_bytes(a).val.0,
15974 idx,
15975 Bytes::to_bytes(b).val.0,
15976 );
15977 Bytes::from_bytes(u8x64 {
15978 val: crate::support::Aligned512(result),
15979 simd: token,
15980 })
15981 }
15982 );
15983 kernel(self, a, b, SHIFT)
15984 }
15985 #[inline(always)]
15986 fn slide_within_blocks_i8x64<const SHIFT: usize>(
15987 self,
15988 a: i8x64<Self>,
15989 b: i8x64<Self>,
15990 ) -> i8x64<Self> {
15991 if SHIFT == 0 {
15992 return a;
15993 }
15994 if SHIFT >= 16usize {
15995 return b;
15996 }
15997 let a = Bytes::to_bytes(a).val.0;
15998 let b = Bytes::to_bytes(b).val.0;
15999 let result = dyn_alignr_512(self, b, a, SHIFT);
16000 Bytes::from_bytes(u8x64 {
16001 val: crate::support::Aligned512(result),
16002 simd: self,
16003 })
16004 }
16005 #[inline(always)]
16006 fn compress_i8x64(self, values: i8x64<Self>, mask: mask8x64<Self>) -> i8x64<Self> {
16007 crate::kernel!(
16008 #[inline(always)]
16009 fn kernel(
16010 token: Avx512,
16011 values: i8x64<Avx512>,
16012 mask: mask8x64<Avx512>,
16013 ) -> i8x64<Avx512> {
16014 _mm512_maskz_compress_epi8(u64::from((mask).val) as u64, values.into())
16015 .simd_into(token)
16016 }
16017 );
16018 kernel(self, values, mask)
16019 }
16020 #[inline(always)]
16021 fn compress_merge_i8x64(
16022 self,
16023 values: i8x64<Self>,
16024 mask: mask8x64<Self>,
16025 merge: i8x64<Self>,
16026 ) -> i8x64<Self> {
16027 crate::kernel!(
16028 #[inline(always)]
16029 fn kernel(
16030 token: Avx512,
16031 values: i8x64<Avx512>,
16032 mask: mask8x64<Avx512>,
16033 merge: i8x64<Avx512>,
16034 ) -> i8x64<Avx512> {
16035 _mm512_mask_compress_epi8(merge.into(), u64::from((mask).val) as u64, values.into())
16036 .simd_into(token)
16037 }
16038 );
16039 kernel(self, values, mask, merge)
16040 }
16041 #[inline(always)]
16042 fn expand_i8x64(self, values: i8x64<Self>, mask: mask8x64<Self>) -> i8x64<Self> {
16043 crate::kernel!(
16044 #[inline(always)]
16045 fn kernel(
16046 token: Avx512,
16047 values: i8x64<Avx512>,
16048 mask: mask8x64<Avx512>,
16049 ) -> i8x64<Avx512> {
16050 _mm512_maskz_expand_epi8(u64::from((mask).val) as u64, values.into())
16051 .simd_into(token)
16052 }
16053 );
16054 kernel(self, values, mask)
16055 }
16056 #[inline(always)]
16057 fn expand_merge_i8x64(
16058 self,
16059 values: i8x64<Self>,
16060 mask: mask8x64<Self>,
16061 merge: i8x64<Self>,
16062 ) -> i8x64<Self> {
16063 crate::kernel!(
16064 #[inline(always)]
16065 fn kernel(
16066 token: Avx512,
16067 values: i8x64<Avx512>,
16068 mask: mask8x64<Avx512>,
16069 merge: i8x64<Avx512>,
16070 ) -> i8x64<Avx512> {
16071 _mm512_mask_expand_epi8(merge.into(), u64::from((mask).val) as u64, values.into())
16072 .simd_into(token)
16073 }
16074 );
16075 kernel(self, values, mask, merge)
16076 }
16077 #[inline(always)]
16078 fn count_ones_i8x64(self, a: i8x64<Self>) -> i8x64<Self> {
16079 crate::kernel!(
16080 #[inline(always)]
16081 fn kernel(token: Avx512, a: i8x64<Avx512>) -> i8x64<Avx512> {
16082 _mm512_popcnt_epi8(a.into()).simd_into(token)
16083 }
16084 );
16085 kernel(self, a)
16086 }
16087 #[inline(always)]
16088 fn count_zeros_i8x64(self, a: i8x64<Self>) -> i8x64<Self> {
16089 self.count_ones_i8x64(self.not_i8x64(a))
16090 }
16091 #[inline(always)]
16092 fn add_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16093 crate::kernel!(
16094 #[inline(always)]
16095 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16096 _mm512_add_epi8(a.into(), b.into()).simd_into(token)
16097 }
16098 );
16099 kernel(self, a, b)
16100 }
16101 #[inline(always)]
16102 fn saturating_add_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16103 crate::kernel!(
16104 #[inline(always)]
16105 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16106 _mm512_adds_epi8(a.into(), b.into()).simd_into(token)
16107 }
16108 );
16109 kernel(self, a, b)
16110 }
16111 #[inline(always)]
16112 fn sub_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16113 crate::kernel!(
16114 #[inline(always)]
16115 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16116 _mm512_sub_epi8(a.into(), b.into()).simd_into(token)
16117 }
16118 );
16119 kernel(self, a, b)
16120 }
16121 #[inline(always)]
16122 fn saturating_sub_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16123 crate::kernel!(
16124 #[inline(always)]
16125 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16126 _mm512_subs_epi8(a.into(), b.into()).simd_into(token)
16127 }
16128 );
16129 kernel(self, a, b)
16130 }
16131 #[inline(always)]
16132 fn mul_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16133 crate::kernel!(
16134 #[inline(always)]
16135 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16136 let a = a.into();
16137 let b = b.into();
16138 let low_mask = _mm512_set1_epi16(0xFF);
16139 let dst_even = _mm512_mullo_epi16(a, b);
16140 let dst_odd = _mm512_maddubs_epi16(a, _mm512_andnot_si512(low_mask, b));
16141 _mm512_or_si512(
16142 _mm512_slli_epi16(dst_odd, 8),
16143 _mm512_and_si512(dst_even, low_mask),
16144 )
16145 .simd_into(token)
16146 }
16147 );
16148 kernel(self, a, b)
16149 }
16150 #[inline(always)]
16151 fn and_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16152 crate::kernel!(
16153 #[inline(always)]
16154 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16155 _mm512_and_si512(a.into(), b.into()).simd_into(token)
16156 }
16157 );
16158 kernel(self, a, b)
16159 }
16160 #[inline(always)]
16161 fn or_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16162 crate::kernel!(
16163 #[inline(always)]
16164 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16165 _mm512_or_si512(a.into(), b.into()).simd_into(token)
16166 }
16167 );
16168 kernel(self, a, b)
16169 }
16170 #[inline(always)]
16171 fn xor_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16172 crate::kernel!(
16173 #[inline(always)]
16174 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16175 _mm512_xor_si512(a.into(), b.into()).simd_into(token)
16176 }
16177 );
16178 kernel(self, a, b)
16179 }
16180 #[inline(always)]
16181 fn not_i8x64(self, a: i8x64<Self>) -> i8x64<Self> {
16182 a ^ !0
16183 }
16184 #[inline(always)]
16185 fn shl_i8x64(self, a: i8x64<Self>, shift: u32) -> i8x64<Self> {
16186 crate::kernel!(
16187 #[inline(always)]
16188 fn kernel(token: Avx512, a: i8x64<Avx512>, shift: u32) -> i8x64<Avx512> {
16189 let val = a.into();
16190 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
16191 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
16192 let byte_mask = _mm512_set1_epi8(mask_byte);
16193 _mm512_sll_epi16(_mm512_and_si512(val, byte_mask), shift_count).simd_into(token)
16194 }
16195 );
16196 kernel(self, a, shift)
16197 }
16198 #[inline(always)]
16199 fn shlv_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16200 crate::kernel!(
16201 #[inline(always)]
16202 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16203 let val = a.into();
16204 let counts = b.into();
16205 let byte_mask = _mm512_set1_epi16(0x00ff);
16206 let lo_counts = _mm512_and_si512(counts, byte_mask);
16207 let hi_counts = _mm512_srli_epi16::<8>(counts);
16208 let lo_shifted = _mm512_sllv_epi16(val, lo_counts);
16209 let hi_values = _mm512_andnot_si512(byte_mask, val);
16210 let hi_shifted = _mm512_sllv_epi16(hi_values, hi_counts);
16211 _mm512_mask_blend_epi8(0xaaaa_aaaa_aaaa_aaaa_u64, lo_shifted, hi_shifted)
16212 .simd_into(token)
16213 }
16214 );
16215 kernel(self, a, b)
16216 }
16217 #[inline(always)]
16218 fn shr_i8x64(self, a: i8x64<Self>, shift: u32) -> i8x64<Self> {
16219 crate::kernel!(
16220 #[inline(always)]
16221 fn kernel(token: Avx512, a: i8x64<Avx512>, shift: u32) -> i8x64<Avx512> {
16222 let val = a.into();
16223 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
16224 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
16225 let byte_mask = _mm512_set1_epi8(mask_byte);
16226 let shifted = _mm512_srl_epi16(val, shift_count);
16227 let result = {
16228 let sign =
16229 _mm512_movm_epi8(_mm512_cmpgt_epi8_mask(_mm512_setzero_si512(), val));
16230 _mm512_ternarylogic_epi32::<0xca>(byte_mask, shifted, sign)
16231 };
16232 result.simd_into(token)
16233 }
16234 );
16235 kernel(self, a, shift)
16236 }
16237 #[inline(always)]
16238 fn shrv_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16239 crate::kernel!(
16240 #[inline(always)]
16241 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16242 let val = a.into();
16243 let counts = b.into();
16244 let zero = _mm512_setzero_si512();
16245 let value_extend = _mm512_movm_epi8(_mm512_cmpgt_epi8_mask(zero, val));
16246 let lo_values = _mm512_unpacklo_epi8(val, value_extend);
16247 let hi_values = _mm512_unpackhi_epi8(val, value_extend);
16248 let lo_counts = _mm512_unpacklo_epi8(counts, zero);
16249 let hi_counts = _mm512_unpackhi_epi8(counts, zero);
16250 let byte_mask = _mm512_set1_epi16(0x00ff);
16251 let lo_shifted =
16252 _mm512_and_si512(_mm512_srav_epi16(lo_values, lo_counts), byte_mask);
16253 let hi_shifted =
16254 _mm512_and_si512(_mm512_srav_epi16(hi_values, hi_counts), byte_mask);
16255 _mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
16256 }
16257 );
16258 kernel(self, a, b)
16259 }
16260 #[inline(always)]
16261 fn max_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16262 crate::kernel!(
16263 #[inline(always)]
16264 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16265 _mm512_max_epi8(a.into(), b.into()).simd_into(token)
16266 }
16267 );
16268 kernel(self, a, b)
16269 }
16270 #[inline(always)]
16271 fn min_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16272 crate::kernel!(
16273 #[inline(always)]
16274 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16275 _mm512_min_epi8(a.into(), b.into()).simd_into(token)
16276 }
16277 );
16278 kernel(self, a, b)
16279 }
16280 #[inline(always)]
16281 fn clamp_i8x64(self, a: i8x64<Self>, b: i8x64<Self>, c: i8x64<Self>) -> i8x64<Self> {
16282 debug_assert!(
16283 b.simd_le(c).all_true(),
16284 "clamp requires min <= max for all elements"
16285 );
16286 a.max(b).min(c)
16287 }
16288 #[inline(always)]
16289 fn simd_eq_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
16290 crate::kernel!(
16291 #[inline(always)]
16292 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> mask8x64<Avx512> {
16293 mask8x64 {
16294 val: _mm512_cmpeq_epi8_mask(a.into(), b.into()),
16295 simd: token,
16296 }
16297 }
16298 );
16299 kernel(self, a, b)
16300 }
16301 #[inline(always)]
16302 fn simd_ne_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
16303 crate::kernel!(
16304 #[inline(always)]
16305 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> mask8x64<Avx512> {
16306 mask8x64 {
16307 val: _mm512_cmpneq_epi8_mask(a.into(), b.into()),
16308 simd: token,
16309 }
16310 }
16311 );
16312 kernel(self, a, b)
16313 }
16314 #[inline(always)]
16315 fn simd_lt_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
16316 crate::kernel!(
16317 #[inline(always)]
16318 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> mask8x64<Avx512> {
16319 mask8x64 {
16320 val: _mm512_cmplt_epi8_mask(a.into(), b.into()),
16321 simd: token,
16322 }
16323 }
16324 );
16325 kernel(self, a, b)
16326 }
16327 #[inline(always)]
16328 fn simd_le_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
16329 crate::kernel!(
16330 #[inline(always)]
16331 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> mask8x64<Avx512> {
16332 mask8x64 {
16333 val: _mm512_cmple_epi8_mask(a.into(), b.into()),
16334 simd: token,
16335 }
16336 }
16337 );
16338 kernel(self, a, b)
16339 }
16340 #[inline(always)]
16341 fn zip_low_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16342 crate::kernel!(
16343 #[inline(always)]
16344 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16345 _mm512_permutex2var_epi8(
16346 a.into(),
16347 _mm512_set_epi8(
16348 95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, 86,
16349 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13,
16350 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3,
16351 66, 2, 65, 1, 64, 0,
16352 ),
16353 b.into(),
16354 )
16355 .simd_into(token)
16356 }
16357 );
16358 kernel(self, a, b)
16359 }
16360 #[inline(always)]
16361 fn zip_high_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16362 crate::kernel!(
16363 #[inline(always)]
16364 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16365 _mm512_permutex2var_epi8(
16366 a.into(),
16367 _mm512_set_epi8(
16368 127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56,
16369 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48,
16370 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40,
16371 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32,
16372 ),
16373 b.into(),
16374 )
16375 .simd_into(token)
16376 }
16377 );
16378 kernel(self, a, b)
16379 }
16380 #[inline(always)]
16381 fn unzip_low_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16382 crate::kernel!(
16383 #[inline(always)]
16384 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16385 _mm512_permutex2var_epi8(
16386 a.into(),
16387 _mm512_set_epi8(
16388 126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100, 98,
16389 96, 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64, 62, 60,
16390 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26, 24, 22,
16391 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
16392 ),
16393 b.into(),
16394 )
16395 .simd_into(token)
16396 }
16397 );
16398 kernel(self, a, b)
16399 }
16400 #[inline(always)]
16401 fn unzip_high_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
16402 crate::kernel!(
16403 #[inline(always)]
16404 fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
16405 _mm512_permutex2var_epi8(
16406 a.into(),
16407 _mm512_set_epi8(
16408 127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101, 99,
16409 97, 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65, 63, 61,
16410 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, 27, 25, 23,
16411 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
16412 ),
16413 b.into(),
16414 )
16415 .simd_into(token)
16416 }
16417 );
16418 kernel(self, a, b)
16419 }
16420 #[inline(always)]
16421 fn interleave_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> (i8x64<Self>, i8x64<Self>) {
16422 crate::kernel!(
16423 #[inline(always)]
16424 fn kernel(
16425 token: Avx512,
16426 a: i8x64<Avx512>,
16427 b: i8x64<Avx512>,
16428 ) -> (i8x64<Avx512>, i8x64<Avx512>) {
16429 let a = a.into();
16430 let b = b.into();
16431 (
16432 _mm512_permutex2var_epi8(
16433 a,
16434 _mm512_set_epi8(
16435 95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23,
16436 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14,
16437 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68,
16438 4, 67, 3, 66, 2, 65, 1, 64, 0,
16439 ),
16440 b,
16441 )
16442 .simd_into(token),
16443 _mm512_permutex2var_epi8(
16444 a,
16445 _mm512_set_epi8(
16446 127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56,
16447 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48,
16448 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40,
16449 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32,
16450 ),
16451 b,
16452 )
16453 .simd_into(token),
16454 )
16455 }
16456 );
16457 kernel(self, a, b)
16458 }
16459 #[inline(always)]
16460 fn deinterleave_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> (i8x64<Self>, i8x64<Self>) {
16461 crate::kernel!(
16462 #[inline(always)]
16463 fn kernel(
16464 token: Avx512,
16465 a: i8x64<Avx512>,
16466 b: i8x64<Avx512>,
16467 ) -> (i8x64<Avx512>, i8x64<Avx512>) {
16468 let a = a.into();
16469 let b = b.into();
16470 (
16471 _mm512_permutex2var_epi8(
16472 a,
16473 _mm512_set_epi8(
16474 126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100,
16475 98, 96, 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64,
16476 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28,
16477 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
16478 ),
16479 b,
16480 )
16481 .simd_into(token),
16482 _mm512_permutex2var_epi8(
16483 a,
16484 _mm512_set_epi8(
16485 127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101,
16486 99, 97, 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65,
16487 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29,
16488 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
16489 ),
16490 b,
16491 )
16492 .simd_into(token),
16493 )
16494 }
16495 );
16496 kernel(self, a, b)
16497 }
16498 #[inline(always)]
16499 fn select_i8x64(self, a: mask8x64<Self>, b: i8x64<Self>, c: i8x64<Self>) -> i8x64<Self> {
16500 crate::kernel!(
16501 #[inline(always)]
16502 fn kernel(
16503 token: Avx512,
16504 a: mask8x64<Avx512>,
16505 b: i8x64<Avx512>,
16506 c: i8x64<Avx512>,
16507 ) -> i8x64<Avx512> {
16508 _mm512_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token)
16509 }
16510 );
16511 kernel(self, a, b, c)
16512 }
16513 #[inline(always)]
16514 fn split_i8x64(self, a: i8x64<Self>) -> (i8x32<Self>, i8x32<Self>) {
16515 crate::kernel!(
16516 #[inline(always)]
16517 fn kernel(token: Avx512, a: i8x64<Avx512>) -> (i8x32<Avx512>, i8x32<Avx512>) {
16518 (
16519 _mm512_castsi512_si256(a.into()).simd_into(token),
16520 _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
16521 )
16522 }
16523 );
16524 kernel(self, a)
16525 }
16526 #[inline(always)]
16527 fn neg_i8x64(self, a: i8x64<Self>) -> i8x64<Self> {
16528 crate::kernel!(
16529 #[inline(always)]
16530 fn kernel(token: Avx512, a: i8x64<Avx512>) -> i8x64<Avx512> {
16531 _mm512_sub_epi8(_mm512_setzero_si512(), a.into()).simd_into(token)
16532 }
16533 );
16534 kernel(self, a)
16535 }
16536 #[inline(always)]
16537 fn widen_i8x64(self, a: i8x64<Self>) -> (i16x32<Self>, i16x32<Self>) {
16538 crate::kernel!(
16539 #[inline(always)]
16540 fn kernel(token: Avx512, a: i8x64<Avx512>) -> (i16x32<Avx512>, i16x32<Avx512>) {
16541 let raw = a.into();
16542 (
16543 _mm512_cvtepi8_epi16(_mm512_castsi512_si256(raw)).simd_into(token),
16544 _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64::<1>(raw)).simd_into(token),
16545 )
16546 }
16547 );
16548 kernel(self, a)
16549 }
16550 #[inline(always)]
16551 fn widening_mul_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> (i16x32<Self>, i16x32<Self>) {
16552 let (a0, a1) = self.widen_i8x64(a);
16553 let (b0, b1) = self.widen_i8x64(b);
16554 (self.mul_i16x32(a0, b0), self.mul_i16x32(a1, b1))
16555 }
16556 #[inline(always)]
16557 fn widening_add_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> (i16x32<Self>, i16x32<Self>) {
16558 let (a0, a1) = self.widen_i8x64(a);
16559 let (b0, b1) = self.widen_i8x64(b);
16560 (self.add_i16x32(a0, b0), self.add_i16x32(a1, b1))
16561 }
16562 #[inline(always)]
16563 fn pairwise_widening_add_i8x64(self, a: i8x64<Self>) -> i16x32<Self> {
16564 crate::kernel!(
16565 #[inline(always)]
16566 fn kernel(token: Avx512, a: i8x64<Avx512>) -> i16x32<Avx512> {
16567 ({ _mm512_maddubs_epi16(_mm512_set1_epi8(1), a.into()) }).simd_into(token)
16568 }
16569 );
16570 kernel(self, a)
16571 }
16572 #[inline(always)]
16573 fn pairwise_dot_product_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i16x32<Self> {
16574 let (low, high) = self.widening_mul_i8x64(a, b);
16575 let (even, odd) = self.deinterleave_i16x32(low, high);
16576 self.add_i16x32(even, odd)
16577 }
16578 #[inline(always)]
16579 fn splat_u8x64(self, val: u8) -> u8x64<Self> {
16580 crate::kernel!(
16581 #[inline(always)]
16582 fn kernel(token: Avx512, val: u8) -> u8x64<Avx512> {
16583 _mm512_set1_epi8(val.cast_signed()).simd_into(token)
16584 }
16585 );
16586 kernel(self, val)
16587 }
16588 #[inline(always)]
16589 fn reverse_u8x64(self, a: u8x64<Self>) -> u8x64<Self> {
16590 let indices: u8x64<Self> = [
16591 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42,
16592 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20,
16593 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
16594 ]
16595 .simd_into(self);
16596 self.swizzle_dyn_u8x64(a, indices)
16597 }
16598 #[inline(always)]
16599 fn slide_u8x64<const SHIFT: usize>(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
16600 crate::kernel!(
16601 #[inline(always)]
16602 fn kernel(
16603 token: Avx512,
16604 a: u8x64<Avx512>,
16605 b: u8x64<Avx512>,
16606 shift: usize,
16607 ) -> u8x64<Avx512> {
16608 if shift >= 64usize {
16609 return b;
16610 }
16611 let idx = _mm512_add_epi8(
16612 _mm512_set_epi8(
16613 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
16614 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
16615 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
16616 5, 4, 3, 2, 1, 0,
16617 ),
16618 _mm512_set1_epi8((shift) as i8),
16619 );
16620 let result = _mm512_permutex2var_epi8(
16621 Bytes::to_bytes(a).val.0,
16622 idx,
16623 Bytes::to_bytes(b).val.0,
16624 );
16625 Bytes::from_bytes(u8x64 {
16626 val: crate::support::Aligned512(result),
16627 simd: token,
16628 })
16629 }
16630 );
16631 kernel(self, a, b, SHIFT)
16632 }
16633 #[inline(always)]
16634 fn slide_within_blocks_u8x64<const SHIFT: usize>(
16635 self,
16636 a: u8x64<Self>,
16637 b: u8x64<Self>,
16638 ) -> u8x64<Self> {
16639 if SHIFT == 0 {
16640 return a;
16641 }
16642 if SHIFT >= 16usize {
16643 return b;
16644 }
16645 let a = Bytes::to_bytes(a).val.0;
16646 let b = Bytes::to_bytes(b).val.0;
16647 let result = dyn_alignr_512(self, b, a, SHIFT);
16648 Bytes::from_bytes(u8x64 {
16649 val: crate::support::Aligned512(result),
16650 simd: self,
16651 })
16652 }
16653 #[inline(always)]
16654 fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
16655 crate::kernel!(
16656 #[inline(always)]
16657 fn kernel(token: Avx512, a: u8x64<Avx512>, indices: u8x64<Avx512>) -> u8x64<Avx512> {
16658 let result = _mm512_shuffle_epi8(Bytes::to_bytes(a).val.0, indices.into());
16659 Bytes::from_bytes(u8x64 {
16660 val: crate::support::Aligned512(result),
16661 simd: token,
16662 })
16663 }
16664 );
16665 kernel(self, a, indices)
16666 }
16667 #[inline(always)]
16668 fn swizzle_dyn_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
16669 crate::kernel!(
16670 #[inline(always)]
16671 fn kernel(token: Avx512, a: u8x64<Avx512>, indices: u8x64<Avx512>) -> u8x64<Avx512> {
16672 let result = _mm512_permutexvar_epi8(indices.into(), Bytes::to_bytes(a).val.0);
16673 Bytes::from_bytes(u8x64 {
16674 val: crate::support::Aligned512(result),
16675 simd: token,
16676 })
16677 }
16678 );
16679 kernel(self, a, indices)
16680 }
16681 #[inline(always)]
16682 fn swizzle_dyn_precise_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
16683 crate::kernel!(
16684 #[inline(always)]
16685 fn kernel(token: Avx512, a: u8x64<Avx512>, indices: u8x64<Avx512>) -> u8x64<Avx512> {
16686 let bytes = Bytes::to_bytes(a).val.0;
16687 let indices = indices.into();
16688 let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64));
16689 let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512());
16690 let result_bytes = u8x64 {
16691 val: crate::support::Aligned512(result),
16692 simd: token,
16693 };
16694 Bytes::from_bytes(result_bytes)
16695 }
16696 );
16697 kernel(self, a, indices)
16698 }
16699 #[inline(always)]
16700 fn concat_swizzle_dyn_u8x64(
16701 self,
16702 a: u8x64<Self>,
16703 b: u8x64<Self>,
16704 indices: u8x64<Self>,
16705 ) -> u8x64<Self> {
16706 crate::kernel!(
16707 #[inline(always)]
16708 fn kernel(
16709 token: Avx512,
16710 a: u8x64<Avx512>,
16711 b: u8x64<Avx512>,
16712 indices: u8x64<Avx512>,
16713 ) -> u8x64<Avx512> {
16714 let result = _mm512_permutex2var_epi8(
16715 Bytes::to_bytes(a).val.0,
16716 indices.into(),
16717 Bytes::to_bytes(b).val.0,
16718 );
16719 Bytes::from_bytes(u8x64 {
16720 val: crate::support::Aligned512(result),
16721 simd: token,
16722 })
16723 }
16724 );
16725 kernel(self, a, b, indices)
16726 }
16727 #[inline(always)]
16728 fn concat_swizzle_dyn_precise_u8x64(
16729 self,
16730 a: u8x64<Self>,
16731 b: u8x64<Self>,
16732 indices: u8x64<Self>,
16733 ) -> u8x64<Self> {
16734 crate::kernel!(
16735 #[inline(always)]
16736 fn kernel(
16737 token: Avx512,
16738 a: u8x64<Avx512>,
16739 b: u8x64<Avx512>,
16740 indices: u8x64<Avx512>,
16741 ) -> u8x64<Avx512> {
16742 let a_bytes = Bytes::to_bytes(a).val.0;
16743 let b_bytes = Bytes::to_bytes(b).val.0;
16744 let indices = indices.into();
16745 let in_range = _mm512_cmplt_epu8_mask(indices, _mm512_set1_epi8(-128));
16746 let result = _mm512_maskz_permutex2var_epi8(in_range, a_bytes, indices, b_bytes);
16747 let result_bytes = u8x64 {
16748 val: crate::support::Aligned512(result),
16749 simd: token,
16750 };
16751 Bytes::from_bytes(result_bytes)
16752 }
16753 );
16754 kernel(self, a, b, indices)
16755 }
16756 #[inline(always)]
16757 fn compress_u8x64(self, values: u8x64<Self>, mask: mask8x64<Self>) -> u8x64<Self> {
16758 crate::kernel!(
16759 #[inline(always)]
16760 fn kernel(
16761 token: Avx512,
16762 values: u8x64<Avx512>,
16763 mask: mask8x64<Avx512>,
16764 ) -> u8x64<Avx512> {
16765 _mm512_maskz_compress_epi8(u64::from((mask).val) as u64, values.into())
16766 .simd_into(token)
16767 }
16768 );
16769 kernel(self, values, mask)
16770 }
16771 #[inline(always)]
16772 fn compress_merge_u8x64(
16773 self,
16774 values: u8x64<Self>,
16775 mask: mask8x64<Self>,
16776 merge: u8x64<Self>,
16777 ) -> u8x64<Self> {
16778 crate::kernel!(
16779 #[inline(always)]
16780 fn kernel(
16781 token: Avx512,
16782 values: u8x64<Avx512>,
16783 mask: mask8x64<Avx512>,
16784 merge: u8x64<Avx512>,
16785 ) -> u8x64<Avx512> {
16786 _mm512_mask_compress_epi8(merge.into(), u64::from((mask).val) as u64, values.into())
16787 .simd_into(token)
16788 }
16789 );
16790 kernel(self, values, mask, merge)
16791 }
16792 #[inline(always)]
16793 fn expand_u8x64(self, values: u8x64<Self>, mask: mask8x64<Self>) -> u8x64<Self> {
16794 crate::kernel!(
16795 #[inline(always)]
16796 fn kernel(
16797 token: Avx512,
16798 values: u8x64<Avx512>,
16799 mask: mask8x64<Avx512>,
16800 ) -> u8x64<Avx512> {
16801 _mm512_maskz_expand_epi8(u64::from((mask).val) as u64, values.into())
16802 .simd_into(token)
16803 }
16804 );
16805 kernel(self, values, mask)
16806 }
16807 #[inline(always)]
16808 fn expand_merge_u8x64(
16809 self,
16810 values: u8x64<Self>,
16811 mask: mask8x64<Self>,
16812 merge: u8x64<Self>,
16813 ) -> u8x64<Self> {
16814 crate::kernel!(
16815 #[inline(always)]
16816 fn kernel(
16817 token: Avx512,
16818 values: u8x64<Avx512>,
16819 mask: mask8x64<Avx512>,
16820 merge: u8x64<Avx512>,
16821 ) -> u8x64<Avx512> {
16822 _mm512_mask_expand_epi8(merge.into(), u64::from((mask).val) as u64, values.into())
16823 .simd_into(token)
16824 }
16825 );
16826 kernel(self, values, mask, merge)
16827 }
16828 #[inline(always)]
16829 fn count_ones_u8x64(self, a: u8x64<Self>) -> u8x64<Self> {
16830 crate::kernel!(
16831 #[inline(always)]
16832 fn kernel(token: Avx512, a: u8x64<Avx512>) -> u8x64<Avx512> {
16833 _mm512_popcnt_epi8(a.into()).simd_into(token)
16834 }
16835 );
16836 kernel(self, a)
16837 }
16838 #[inline(always)]
16839 fn count_zeros_u8x64(self, a: u8x64<Self>) -> u8x64<Self> {
16840 self.count_ones_u8x64(self.not_u8x64(a))
16841 }
16842 #[inline(always)]
16843 fn add_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
16844 crate::kernel!(
16845 #[inline(always)]
16846 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
16847 _mm512_add_epi8(a.into(), b.into()).simd_into(token)
16848 }
16849 );
16850 kernel(self, a, b)
16851 }
16852 #[inline(always)]
16853 fn saturating_add_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
16854 crate::kernel!(
16855 #[inline(always)]
16856 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
16857 _mm512_adds_epu8(a.into(), b.into()).simd_into(token)
16858 }
16859 );
16860 kernel(self, a, b)
16861 }
16862 #[inline(always)]
16863 fn sub_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
16864 crate::kernel!(
16865 #[inline(always)]
16866 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
16867 _mm512_sub_epi8(a.into(), b.into()).simd_into(token)
16868 }
16869 );
16870 kernel(self, a, b)
16871 }
16872 #[inline(always)]
16873 fn saturating_sub_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
16874 crate::kernel!(
16875 #[inline(always)]
16876 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
16877 _mm512_subs_epu8(a.into(), b.into()).simd_into(token)
16878 }
16879 );
16880 kernel(self, a, b)
16881 }
16882 #[inline(always)]
16883 fn mul_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
16884 crate::kernel!(
16885 #[inline(always)]
16886 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
16887 let a = a.into();
16888 let b = b.into();
16889 let low_mask = _mm512_set1_epi16(0xFF);
16890 let dst_even = _mm512_mullo_epi16(a, b);
16891 let dst_odd = _mm512_maddubs_epi16(a, _mm512_andnot_si512(low_mask, b));
16892 _mm512_or_si512(
16893 _mm512_slli_epi16(dst_odd, 8),
16894 _mm512_and_si512(dst_even, low_mask),
16895 )
16896 .simd_into(token)
16897 }
16898 );
16899 kernel(self, a, b)
16900 }
16901 #[inline(always)]
16902 fn and_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
16903 crate::kernel!(
16904 #[inline(always)]
16905 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
16906 _mm512_and_si512(a.into(), b.into()).simd_into(token)
16907 }
16908 );
16909 kernel(self, a, b)
16910 }
16911 #[inline(always)]
16912 fn or_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
16913 crate::kernel!(
16914 #[inline(always)]
16915 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
16916 _mm512_or_si512(a.into(), b.into()).simd_into(token)
16917 }
16918 );
16919 kernel(self, a, b)
16920 }
16921 #[inline(always)]
16922 fn xor_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
16923 crate::kernel!(
16924 #[inline(always)]
16925 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
16926 _mm512_xor_si512(a.into(), b.into()).simd_into(token)
16927 }
16928 );
16929 kernel(self, a, b)
16930 }
16931 #[inline(always)]
16932 fn not_u8x64(self, a: u8x64<Self>) -> u8x64<Self> {
16933 a ^ !0
16934 }
16935 #[inline(always)]
16936 fn shl_u8x64(self, a: u8x64<Self>, shift: u32) -> u8x64<Self> {
16937 crate::kernel!(
16938 #[inline(always)]
16939 fn kernel(token: Avx512, a: u8x64<Avx512>, shift: u32) -> u8x64<Avx512> {
16940 let val = a.into();
16941 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
16942 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
16943 let byte_mask = _mm512_set1_epi8(mask_byte);
16944 _mm512_sll_epi16(_mm512_and_si512(val, byte_mask), shift_count).simd_into(token)
16945 }
16946 );
16947 kernel(self, a, shift)
16948 }
16949 #[inline(always)]
16950 fn shlv_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
16951 crate::kernel!(
16952 #[inline(always)]
16953 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
16954 let val = a.into();
16955 let counts = b.into();
16956 let byte_mask = _mm512_set1_epi16(0x00ff);
16957 let lo_counts = _mm512_and_si512(counts, byte_mask);
16958 let hi_counts = _mm512_srli_epi16::<8>(counts);
16959 let lo_shifted = _mm512_sllv_epi16(val, lo_counts);
16960 let hi_values = _mm512_andnot_si512(byte_mask, val);
16961 let hi_shifted = _mm512_sllv_epi16(hi_values, hi_counts);
16962 _mm512_mask_blend_epi8(0xaaaa_aaaa_aaaa_aaaa_u64, lo_shifted, hi_shifted)
16963 .simd_into(token)
16964 }
16965 );
16966 kernel(self, a, b)
16967 }
16968 #[inline(always)]
16969 fn shr_u8x64(self, a: u8x64<Self>, shift: u32) -> u8x64<Self> {
16970 crate::kernel!(
16971 #[inline(always)]
16972 fn kernel(token: Avx512, a: u8x64<Avx512>, shift: u32) -> u8x64<Avx512> {
16973 let val = a.into();
16974 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
16975 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
16976 let byte_mask = _mm512_set1_epi8(mask_byte);
16977 let shifted = _mm512_srl_epi16(val, shift_count);
16978 let result = { _mm512_and_si512(shifted, byte_mask) };
16979 result.simd_into(token)
16980 }
16981 );
16982 kernel(self, a, shift)
16983 }
16984 #[inline(always)]
16985 fn shrv_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
16986 crate::kernel!(
16987 #[inline(always)]
16988 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
16989 let val = a.into();
16990 let counts = b.into();
16991 let zero = _mm512_setzero_si512();
16992 let value_extend = zero;
16993 let lo_values = _mm512_unpacklo_epi8(val, value_extend);
16994 let hi_values = _mm512_unpackhi_epi8(val, value_extend);
16995 let lo_counts = _mm512_unpacklo_epi8(counts, zero);
16996 let hi_counts = _mm512_unpackhi_epi8(counts, zero);
16997 let byte_mask = _mm512_set1_epi16(0x00ff);
16998 let lo_shifted =
16999 _mm512_and_si512(_mm512_srlv_epi16(lo_values, lo_counts), byte_mask);
17000 let hi_shifted =
17001 _mm512_and_si512(_mm512_srlv_epi16(hi_values, hi_counts), byte_mask);
17002 _mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
17003 }
17004 );
17005 kernel(self, a, b)
17006 }
17007 #[inline(always)]
17008 fn max_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
17009 crate::kernel!(
17010 #[inline(always)]
17011 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
17012 _mm512_max_epu8(a.into(), b.into()).simd_into(token)
17013 }
17014 );
17015 kernel(self, a, b)
17016 }
17017 #[inline(always)]
17018 fn min_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
17019 crate::kernel!(
17020 #[inline(always)]
17021 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
17022 _mm512_min_epu8(a.into(), b.into()).simd_into(token)
17023 }
17024 );
17025 kernel(self, a, b)
17026 }
17027 #[inline(always)]
17028 fn clamp_u8x64(self, a: u8x64<Self>, b: u8x64<Self>, c: u8x64<Self>) -> u8x64<Self> {
17029 debug_assert!(
17030 b.simd_le(c).all_true(),
17031 "clamp requires min <= max for all elements"
17032 );
17033 a.max(b).min(c)
17034 }
17035 #[inline(always)]
17036 fn simd_eq_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
17037 crate::kernel!(
17038 #[inline(always)]
17039 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> mask8x64<Avx512> {
17040 mask8x64 {
17041 val: _mm512_cmpeq_epu8_mask(a.into(), b.into()),
17042 simd: token,
17043 }
17044 }
17045 );
17046 kernel(self, a, b)
17047 }
17048 #[inline(always)]
17049 fn simd_ne_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
17050 crate::kernel!(
17051 #[inline(always)]
17052 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> mask8x64<Avx512> {
17053 mask8x64 {
17054 val: _mm512_cmpneq_epu8_mask(a.into(), b.into()),
17055 simd: token,
17056 }
17057 }
17058 );
17059 kernel(self, a, b)
17060 }
17061 #[inline(always)]
17062 fn simd_lt_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
17063 crate::kernel!(
17064 #[inline(always)]
17065 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> mask8x64<Avx512> {
17066 mask8x64 {
17067 val: _mm512_cmplt_epu8_mask(a.into(), b.into()),
17068 simd: token,
17069 }
17070 }
17071 );
17072 kernel(self, a, b)
17073 }
17074 #[inline(always)]
17075 fn simd_le_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
17076 crate::kernel!(
17077 #[inline(always)]
17078 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> mask8x64<Avx512> {
17079 mask8x64 {
17080 val: _mm512_cmple_epu8_mask(a.into(), b.into()),
17081 simd: token,
17082 }
17083 }
17084 );
17085 kernel(self, a, b)
17086 }
17087 #[inline(always)]
17088 fn zip_low_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
17089 crate::kernel!(
17090 #[inline(always)]
17091 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
17092 _mm512_permutex2var_epi8(
17093 a.into(),
17094 _mm512_set_epi8(
17095 95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, 86,
17096 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13,
17097 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3,
17098 66, 2, 65, 1, 64, 0,
17099 ),
17100 b.into(),
17101 )
17102 .simd_into(token)
17103 }
17104 );
17105 kernel(self, a, b)
17106 }
17107 #[inline(always)]
17108 fn zip_high_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
17109 crate::kernel!(
17110 #[inline(always)]
17111 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
17112 _mm512_permutex2var_epi8(
17113 a.into(),
17114 _mm512_set_epi8(
17115 127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56,
17116 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48,
17117 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40,
17118 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32,
17119 ),
17120 b.into(),
17121 )
17122 .simd_into(token)
17123 }
17124 );
17125 kernel(self, a, b)
17126 }
17127 #[inline(always)]
17128 fn unzip_low_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
17129 crate::kernel!(
17130 #[inline(always)]
17131 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
17132 _mm512_permutex2var_epi8(
17133 a.into(),
17134 _mm512_set_epi8(
17135 126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100, 98,
17136 96, 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64, 62, 60,
17137 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26, 24, 22,
17138 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
17139 ),
17140 b.into(),
17141 )
17142 .simd_into(token)
17143 }
17144 );
17145 kernel(self, a, b)
17146 }
17147 #[inline(always)]
17148 fn unzip_high_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
17149 crate::kernel!(
17150 #[inline(always)]
17151 fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
17152 _mm512_permutex2var_epi8(
17153 a.into(),
17154 _mm512_set_epi8(
17155 127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101, 99,
17156 97, 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65, 63, 61,
17157 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, 27, 25, 23,
17158 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
17159 ),
17160 b.into(),
17161 )
17162 .simd_into(token)
17163 }
17164 );
17165 kernel(self, a, b)
17166 }
17167 #[inline(always)]
17168 fn interleave_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> (u8x64<Self>, u8x64<Self>) {
17169 crate::kernel!(
17170 #[inline(always)]
17171 fn kernel(
17172 token: Avx512,
17173 a: u8x64<Avx512>,
17174 b: u8x64<Avx512>,
17175 ) -> (u8x64<Avx512>, u8x64<Avx512>) {
17176 let a = a.into();
17177 let b = b.into();
17178 (
17179 _mm512_permutex2var_epi8(
17180 a,
17181 _mm512_set_epi8(
17182 95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23,
17183 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14,
17184 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68,
17185 4, 67, 3, 66, 2, 65, 1, 64, 0,
17186 ),
17187 b,
17188 )
17189 .simd_into(token),
17190 _mm512_permutex2var_epi8(
17191 a,
17192 _mm512_set_epi8(
17193 127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56,
17194 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48,
17195 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40,
17196 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32,
17197 ),
17198 b,
17199 )
17200 .simd_into(token),
17201 )
17202 }
17203 );
17204 kernel(self, a, b)
17205 }
17206 #[inline(always)]
17207 fn deinterleave_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> (u8x64<Self>, u8x64<Self>) {
17208 crate::kernel!(
17209 #[inline(always)]
17210 fn kernel(
17211 token: Avx512,
17212 a: u8x64<Avx512>,
17213 b: u8x64<Avx512>,
17214 ) -> (u8x64<Avx512>, u8x64<Avx512>) {
17215 let a = a.into();
17216 let b = b.into();
17217 (
17218 _mm512_permutex2var_epi8(
17219 a,
17220 _mm512_set_epi8(
17221 126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100,
17222 98, 96, 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64,
17223 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28,
17224 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
17225 ),
17226 b,
17227 )
17228 .simd_into(token),
17229 _mm512_permutex2var_epi8(
17230 a,
17231 _mm512_set_epi8(
17232 127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101,
17233 99, 97, 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65,
17234 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29,
17235 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
17236 ),
17237 b,
17238 )
17239 .simd_into(token),
17240 )
17241 }
17242 );
17243 kernel(self, a, b)
17244 }
17245 #[inline(always)]
17246 fn select_u8x64(self, a: mask8x64<Self>, b: u8x64<Self>, c: u8x64<Self>) -> u8x64<Self> {
17247 crate::kernel!(
17248 #[inline(always)]
17249 fn kernel(
17250 token: Avx512,
17251 a: mask8x64<Avx512>,
17252 b: u8x64<Avx512>,
17253 c: u8x64<Avx512>,
17254 ) -> u8x64<Avx512> {
17255 _mm512_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token)
17256 }
17257 );
17258 kernel(self, a, b, c)
17259 }
17260 #[inline(always)]
17261 fn split_u8x64(self, a: u8x64<Self>) -> (u8x32<Self>, u8x32<Self>) {
17262 crate::kernel!(
17263 #[inline(always)]
17264 fn kernel(token: Avx512, a: u8x64<Avx512>) -> (u8x32<Avx512>, u8x32<Avx512>) {
17265 (
17266 _mm512_castsi512_si256(a.into()).simd_into(token),
17267 _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
17268 )
17269 }
17270 );
17271 kernel(self, a)
17272 }
17273 #[inline(always)]
17274 fn widen_u8x64(self, a: u8x64<Self>) -> (u16x32<Self>, u16x32<Self>) {
17275 crate::kernel!(
17276 #[inline(always)]
17277 fn kernel(token: Avx512, a: u8x64<Avx512>) -> (u16x32<Avx512>, u16x32<Avx512>) {
17278 let raw = a.into();
17279 (
17280 _mm512_cvtepu8_epi16(_mm512_castsi512_si256(raw)).simd_into(token),
17281 _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64::<1>(raw)).simd_into(token),
17282 )
17283 }
17284 );
17285 kernel(self, a)
17286 }
17287 #[inline(always)]
17288 fn widening_mul_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> (u16x32<Self>, u16x32<Self>) {
17289 let (a0, a1) = self.widen_u8x64(a);
17290 let (b0, b1) = self.widen_u8x64(b);
17291 (self.mul_u16x32(a0, b0), self.mul_u16x32(a1, b1))
17292 }
17293 #[inline(always)]
17294 fn widening_add_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> (u16x32<Self>, u16x32<Self>) {
17295 let (a0, a1) = self.widen_u8x64(a);
17296 let (b0, b1) = self.widen_u8x64(b);
17297 (self.add_u16x32(a0, b0), self.add_u16x32(a1, b1))
17298 }
17299 #[inline(always)]
17300 fn pairwise_widening_add_u8x64(self, a: u8x64<Self>) -> u16x32<Self> {
17301 crate::kernel!(
17302 #[inline(always)]
17303 fn kernel(token: Avx512, a: u8x64<Avx512>) -> u16x32<Avx512> {
17304 ({ _mm512_maddubs_epi16(a.into(), _mm512_set1_epi8(1)) }).simd_into(token)
17305 }
17306 );
17307 kernel(self, a)
17308 }
17309 #[inline(always)]
17310 fn pairwise_dot_product_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u16x32<Self> {
17311 let (low, high) = self.widening_mul_u8x64(a, b);
17312 let (even, odd) = self.deinterleave_u16x32(low, high);
17313 self.add_u16x32(even, odd)
17314 }
17315 #[inline(always)]
17316 fn splat_mask8x64(self, val: bool) -> mask8x64<Self> {
17317 mask8x64 {
17318 val: if val { u64::MAX } else { 0 },
17319 simd: self,
17320 }
17321 }
17322 #[inline(always)]
17323 fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64<Self> {
17324 mask8x64 {
17325 val: bits & u64::MAX,
17326 simd: self,
17327 }
17328 }
17329 #[inline(always)]
17330 fn to_bitmask_mask8x64(self, a: mask8x64<Self>) -> u64 {
17331 u64::from((a).val) & u64::MAX
17332 }
17333 #[inline(always)]
17334 fn from_vector_mask8x64(self, vector: i8x64<Self>) -> mask8x64<Self> {
17335 crate::kernel!(
17336 #[inline(always)]
17337 fn kernel(token: Avx512, vector: i8x64<Avx512>) -> mask8x64<Avx512> {
17338 _mm512_movepi8_mask(vector.into()).simd_into(token)
17339 }
17340 );
17341 kernel(self, vector)
17342 }
17343 #[inline(always)]
17344 fn to_vector_mask8x64(self, a: mask8x64<Self>) -> i8x64<Self> {
17345 crate::kernel!(
17346 #[inline(always)]
17347 fn kernel(token: Avx512, a: mask8x64<Avx512>) -> i8x64<Avx512> {
17348 _mm512_movm_epi8(a.val).simd_into(token)
17349 }
17350 );
17351 kernel(self, a)
17352 }
17353 #[inline(always)]
17354 fn set_mask8x64(self, a: &mut mask8x64<Self>, index: usize, value: bool) -> () {
17355 assert!(
17356 index < 64usize,
17357 "mask lane index {index} is out of bounds for {} lanes",
17358 64usize
17359 );
17360 let bit = 1u64 << index;
17361 let bits = u64::from((a).val);
17362 let bits = if value { bits | bit } else { bits & !bit };
17363 *a = mask8x64 {
17364 val: bits,
17365 simd: self,
17366 };
17367 }
17368 #[inline(always)]
17369 fn rotate_elements_left_mask8x64<const OFFSET: usize>(
17370 self,
17371 a: mask8x64<Self>,
17372 ) -> mask8x64<Self> {
17373 mask8x64 {
17374 val: a.val.rotate_right((OFFSET % 64) as u32),
17375 simd: self,
17376 }
17377 }
17378 #[inline(always)]
17379 fn rotate_elements_right_mask8x64<const OFFSET: usize>(
17380 self,
17381 a: mask8x64<Self>,
17382 ) -> mask8x64<Self> {
17383 mask8x64 {
17384 val: a.val.rotate_left((OFFSET % 64) as u32),
17385 simd: self,
17386 }
17387 }
17388 #[inline(always)]
17389 fn and_mask8x64(self, a: mask8x64<Self>, b: mask8x64<Self>) -> mask8x64<Self> {
17390 mask8x64 {
17391 val: (u64::from((a).val) & u64::from((b).val)) & u64::MAX,
17392 simd: self,
17393 }
17394 }
17395 #[inline(always)]
17396 fn or_mask8x64(self, a: mask8x64<Self>, b: mask8x64<Self>) -> mask8x64<Self> {
17397 mask8x64 {
17398 val: (u64::from((a).val) | u64::from((b).val)) & u64::MAX,
17399 simd: self,
17400 }
17401 }
17402 #[inline(always)]
17403 fn xor_mask8x64(self, a: mask8x64<Self>, b: mask8x64<Self>) -> mask8x64<Self> {
17404 mask8x64 {
17405 val: (u64::from((a).val) ^ u64::from((b).val)) & u64::MAX,
17406 simd: self,
17407 }
17408 }
17409 #[inline(always)]
17410 fn not_mask8x64(self, a: mask8x64<Self>) -> mask8x64<Self> {
17411 mask8x64 {
17412 val: (!u64::from((a).val)) & u64::MAX,
17413 simd: self,
17414 }
17415 }
17416 #[inline(always)]
17417 fn reverse_mask8x64(self, a: mask8x64<Self>) -> mask8x64<Self> {
17418 mask8x64 {
17419 val: a.val.reverse_bits() >> 0usize,
17420 simd: self,
17421 }
17422 }
17423 #[inline(always)]
17424 fn select_mask8x64(
17425 self,
17426 a: mask8x64<Self>,
17427 b: mask8x64<Self>,
17428 c: mask8x64<Self>,
17429 ) -> mask8x64<Self> {
17430 mask8x64 {
17431 val: ((u64::from((a).val) & u64::from((b).val))
17432 | ((!u64::from((a).val)) & u64::from((c).val)))
17433 & u64::MAX,
17434 simd: self,
17435 }
17436 }
17437 #[inline(always)]
17438 fn simd_eq_mask8x64(self, a: mask8x64<Self>, b: mask8x64<Self>) -> mask8x64<Self> {
17439 mask8x64 {
17440 val: !u64::from(a.val ^ b.val) & u64::MAX,
17441 simd: self,
17442 }
17443 }
17444 #[inline(always)]
17445 fn simd_ne_mask8x64(self, a: mask8x64<Self>, b: mask8x64<Self>) -> mask8x64<Self> {
17446 mask8x64 {
17447 val: u64::from(a.val ^ b.val) & u64::MAX,
17448 simd: self,
17449 }
17450 }
17451 #[inline(always)]
17452 fn any_true_mask8x64(self, a: mask8x64<Self>) -> bool {
17453 let bits = u64::from((a).val) & u64::MAX;
17454 bits != 0
17455 }
17456 #[inline(always)]
17457 fn all_true_mask8x64(self, a: mask8x64<Self>) -> bool {
17458 let bits = u64::from((a).val) & u64::MAX;
17459 bits == u64::MAX
17460 }
17461 #[inline(always)]
17462 fn any_false_mask8x64(self, a: mask8x64<Self>) -> bool {
17463 let bits = u64::from((a).val) & u64::MAX;
17464 bits != u64::MAX
17465 }
17466 #[inline(always)]
17467 fn all_false_mask8x64(self, a: mask8x64<Self>) -> bool {
17468 let bits = u64::from((a).val) & u64::MAX;
17469 bits == 0
17470 }
17471 #[inline(always)]
17472 fn split_mask8x64(self, a: mask8x64<Self>) -> (mask8x32<Self>, mask8x32<Self>) {
17473 let bits = u64::from(a.val);
17474 (
17475 mask8x32 {
17476 val: (bits & 4294967295u64) as _,
17477 simd: self,
17478 },
17479 mask8x32 {
17480 val: ((bits >> 32usize) & 4294967295u64) as _,
17481 simd: self,
17482 },
17483 )
17484 }
17485 #[inline(always)]
17486 fn widen_mask8x64(self, a: mask8x64<Self>) -> (mask16x32<Self>, mask16x32<Self>) {
17487 let bits = u64::from(a.val);
17488 (
17489 mask16x32 {
17490 val: (bits & 4294967295u64) as _,
17491 simd: self,
17492 },
17493 mask16x32 {
17494 val: ((bits >> 32usize) & 4294967295u64) as _,
17495 simd: self,
17496 },
17497 )
17498 }
17499 #[inline(always)]
17500 fn abs_i16x32(self, a: i16x32<Self>) -> i16x32<Self> {
17501 crate::kernel!(
17502 #[inline(always)]
17503 fn kernel(token: Avx512, a: i16x32<Avx512>) -> i16x32<Avx512> {
17504 _mm512_abs_epi16(a.into()).simd_into(token)
17505 }
17506 );
17507 kernel(self, a)
17508 }
17509 #[inline(always)]
17510 fn splat_i16x32(self, val: i16) -> i16x32<Self> {
17511 crate::kernel!(
17512 #[inline(always)]
17513 fn kernel(token: Avx512, val: i16) -> i16x32<Avx512> {
17514 _mm512_set1_epi16(val).simd_into(token)
17515 }
17516 );
17517 kernel(self, val)
17518 }
17519 #[inline(always)]
17520 fn reverse_i16x32(self, a: i16x32<Self>) -> i16x32<Self> {
17521 let indices: u8x64<Self> = [
17522 62, 63, 60, 61, 58, 59, 56, 57, 54, 55, 52, 53, 50, 51, 48, 49, 46, 47, 44, 45, 42, 43,
17523 40, 41, 38, 39, 36, 37, 34, 35, 32, 33, 30, 31, 28, 29, 26, 27, 24, 25, 22, 23, 20, 21,
17524 18, 19, 16, 17, 14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1,
17525 ]
17526 .simd_into(self);
17527 self.swizzle_dyn_i16x32(a, indices)
17528 }
17529 #[inline(always)]
17530 fn slide_i16x32<const SHIFT: usize>(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17531 crate::kernel!(
17532 #[inline(always)]
17533 fn kernel(
17534 token: Avx512,
17535 a: i16x32<Avx512>,
17536 b: i16x32<Avx512>,
17537 shift: usize,
17538 ) -> i16x32<Avx512> {
17539 if shift >= 32usize {
17540 return b;
17541 }
17542 let idx = _mm512_add_epi8(
17543 _mm512_set_epi8(
17544 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
17545 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
17546 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
17547 5, 4, 3, 2, 1, 0,
17548 ),
17549 _mm512_set1_epi8((shift * 2usize) as i8),
17550 );
17551 let result = _mm512_permutex2var_epi8(
17552 Bytes::to_bytes(a).val.0,
17553 idx,
17554 Bytes::to_bytes(b).val.0,
17555 );
17556 Bytes::from_bytes(u8x64 {
17557 val: crate::support::Aligned512(result),
17558 simd: token,
17559 })
17560 }
17561 );
17562 kernel(self, a, b, SHIFT)
17563 }
17564 #[inline(always)]
17565 fn slide_within_blocks_i16x32<const SHIFT: usize>(
17566 self,
17567 a: i16x32<Self>,
17568 b: i16x32<Self>,
17569 ) -> i16x32<Self> {
17570 if SHIFT == 0 {
17571 return a;
17572 }
17573 if SHIFT >= 8usize {
17574 return b;
17575 }
17576 let a = Bytes::to_bytes(a).val.0;
17577 let b = Bytes::to_bytes(b).val.0;
17578 let result = dyn_alignr_512(self, b, a, SHIFT * 2usize);
17579 Bytes::from_bytes(u8x64 {
17580 val: crate::support::Aligned512(result),
17581 simd: self,
17582 })
17583 }
17584 #[inline(always)]
17585 fn compress_i16x32(self, values: i16x32<Self>, mask: mask16x32<Self>) -> i16x32<Self> {
17586 crate::kernel!(
17587 #[inline(always)]
17588 fn kernel(
17589 token: Avx512,
17590 values: i16x32<Avx512>,
17591 mask: mask16x32<Avx512>,
17592 ) -> i16x32<Avx512> {
17593 _mm512_maskz_compress_epi16(u64::from((mask).val) as u32, values.into())
17594 .simd_into(token)
17595 }
17596 );
17597 kernel(self, values, mask)
17598 }
17599 #[inline(always)]
17600 fn compress_merge_i16x32(
17601 self,
17602 values: i16x32<Self>,
17603 mask: mask16x32<Self>,
17604 merge: i16x32<Self>,
17605 ) -> i16x32<Self> {
17606 crate::kernel!(
17607 #[inline(always)]
17608 fn kernel(
17609 token: Avx512,
17610 values: i16x32<Avx512>,
17611 mask: mask16x32<Avx512>,
17612 merge: i16x32<Avx512>,
17613 ) -> i16x32<Avx512> {
17614 _mm512_mask_compress_epi16(
17615 merge.into(),
17616 u64::from((mask).val) as u32,
17617 values.into(),
17618 )
17619 .simd_into(token)
17620 }
17621 );
17622 kernel(self, values, mask, merge)
17623 }
17624 #[inline(always)]
17625 fn expand_i16x32(self, values: i16x32<Self>, mask: mask16x32<Self>) -> i16x32<Self> {
17626 crate::kernel!(
17627 #[inline(always)]
17628 fn kernel(
17629 token: Avx512,
17630 values: i16x32<Avx512>,
17631 mask: mask16x32<Avx512>,
17632 ) -> i16x32<Avx512> {
17633 _mm512_maskz_expand_epi16(u64::from((mask).val) as u32, values.into())
17634 .simd_into(token)
17635 }
17636 );
17637 kernel(self, values, mask)
17638 }
17639 #[inline(always)]
17640 fn expand_merge_i16x32(
17641 self,
17642 values: i16x32<Self>,
17643 mask: mask16x32<Self>,
17644 merge: i16x32<Self>,
17645 ) -> i16x32<Self> {
17646 crate::kernel!(
17647 #[inline(always)]
17648 fn kernel(
17649 token: Avx512,
17650 values: i16x32<Avx512>,
17651 mask: mask16x32<Avx512>,
17652 merge: i16x32<Avx512>,
17653 ) -> i16x32<Avx512> {
17654 _mm512_mask_expand_epi16(merge.into(), u64::from((mask).val) as u32, values.into())
17655 .simd_into(token)
17656 }
17657 );
17658 kernel(self, values, mask, merge)
17659 }
17660 #[inline(always)]
17661 fn count_ones_i16x32(self, a: i16x32<Self>) -> i16x32<Self> {
17662 crate::kernel!(
17663 #[inline(always)]
17664 fn kernel(token: Avx512, a: i16x32<Avx512>) -> i16x32<Avx512> {
17665 _mm512_popcnt_epi16(a.into()).simd_into(token)
17666 }
17667 );
17668 kernel(self, a)
17669 }
17670 #[inline(always)]
17671 fn count_zeros_i16x32(self, a: i16x32<Self>) -> i16x32<Self> {
17672 self.count_ones_i16x32(self.not_i16x32(a))
17673 }
17674 #[inline(always)]
17675 fn add_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17676 crate::kernel!(
17677 #[inline(always)]
17678 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17679 _mm512_add_epi16(a.into(), b.into()).simd_into(token)
17680 }
17681 );
17682 kernel(self, a, b)
17683 }
17684 #[inline(always)]
17685 fn saturating_add_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17686 crate::kernel!(
17687 #[inline(always)]
17688 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17689 _mm512_adds_epi16(a.into(), b.into()).simd_into(token)
17690 }
17691 );
17692 kernel(self, a, b)
17693 }
17694 #[inline(always)]
17695 fn sub_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17696 crate::kernel!(
17697 #[inline(always)]
17698 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17699 _mm512_sub_epi16(a.into(), b.into()).simd_into(token)
17700 }
17701 );
17702 kernel(self, a, b)
17703 }
17704 #[inline(always)]
17705 fn saturating_sub_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17706 crate::kernel!(
17707 #[inline(always)]
17708 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17709 _mm512_subs_epi16(a.into(), b.into()).simd_into(token)
17710 }
17711 );
17712 kernel(self, a, b)
17713 }
17714 #[inline(always)]
17715 fn mul_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17716 crate::kernel!(
17717 #[inline(always)]
17718 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17719 _mm512_mullo_epi16(a.into(), b.into()).simd_into(token)
17720 }
17721 );
17722 kernel(self, a, b)
17723 }
17724 #[inline(always)]
17725 fn and_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17726 crate::kernel!(
17727 #[inline(always)]
17728 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17729 _mm512_and_si512(a.into(), b.into()).simd_into(token)
17730 }
17731 );
17732 kernel(self, a, b)
17733 }
17734 #[inline(always)]
17735 fn or_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17736 crate::kernel!(
17737 #[inline(always)]
17738 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17739 _mm512_or_si512(a.into(), b.into()).simd_into(token)
17740 }
17741 );
17742 kernel(self, a, b)
17743 }
17744 #[inline(always)]
17745 fn xor_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17746 crate::kernel!(
17747 #[inline(always)]
17748 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17749 _mm512_xor_si512(a.into(), b.into()).simd_into(token)
17750 }
17751 );
17752 kernel(self, a, b)
17753 }
17754 #[inline(always)]
17755 fn not_i16x32(self, a: i16x32<Self>) -> i16x32<Self> {
17756 a ^ !0
17757 }
17758 #[inline(always)]
17759 fn shl_i16x32(self, a: i16x32<Self>, shift: u32) -> i16x32<Self> {
17760 crate::kernel!(
17761 #[inline(always)]
17762 fn kernel(token: Avx512, a: i16x32<Avx512>, shift: u32) -> i16x32<Avx512> {
17763 _mm512_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
17764 }
17765 );
17766 kernel(self, a, shift)
17767 }
17768 #[inline(always)]
17769 fn shlv_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17770 crate::kernel!(
17771 #[inline(always)]
17772 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17773 _mm512_sllv_epi16(a.into(), b.into()).simd_into(token)
17774 }
17775 );
17776 kernel(self, a, b)
17777 }
17778 #[inline(always)]
17779 fn shr_i16x32(self, a: i16x32<Self>, shift: u32) -> i16x32<Self> {
17780 crate::kernel!(
17781 #[inline(always)]
17782 fn kernel(token: Avx512, a: i16x32<Avx512>, shift: u32) -> i16x32<Avx512> {
17783 _mm512_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
17784 }
17785 );
17786 kernel(self, a, shift)
17787 }
17788 #[inline(always)]
17789 fn shrv_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17790 crate::kernel!(
17791 #[inline(always)]
17792 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17793 _mm512_srav_epi16(a.into(), b.into()).simd_into(token)
17794 }
17795 );
17796 kernel(self, a, b)
17797 }
17798 #[inline(always)]
17799 fn max_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17800 crate::kernel!(
17801 #[inline(always)]
17802 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17803 _mm512_max_epi16(a.into(), b.into()).simd_into(token)
17804 }
17805 );
17806 kernel(self, a, b)
17807 }
17808 #[inline(always)]
17809 fn min_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17810 crate::kernel!(
17811 #[inline(always)]
17812 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17813 _mm512_min_epi16(a.into(), b.into()).simd_into(token)
17814 }
17815 );
17816 kernel(self, a, b)
17817 }
17818 #[inline(always)]
17819 fn clamp_i16x32(self, a: i16x32<Self>, b: i16x32<Self>, c: i16x32<Self>) -> i16x32<Self> {
17820 debug_assert!(
17821 b.simd_le(c).all_true(),
17822 "clamp requires min <= max for all elements"
17823 );
17824 a.max(b).min(c)
17825 }
17826 #[inline(always)]
17827 fn simd_eq_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
17828 crate::kernel!(
17829 #[inline(always)]
17830 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> mask16x32<Avx512> {
17831 mask16x32 {
17832 val: _mm512_cmpeq_epi16_mask(a.into(), b.into()),
17833 simd: token,
17834 }
17835 }
17836 );
17837 kernel(self, a, b)
17838 }
17839 #[inline(always)]
17840 fn simd_ne_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
17841 crate::kernel!(
17842 #[inline(always)]
17843 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> mask16x32<Avx512> {
17844 mask16x32 {
17845 val: _mm512_cmpneq_epi16_mask(a.into(), b.into()),
17846 simd: token,
17847 }
17848 }
17849 );
17850 kernel(self, a, b)
17851 }
17852 #[inline(always)]
17853 fn simd_lt_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
17854 crate::kernel!(
17855 #[inline(always)]
17856 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> mask16x32<Avx512> {
17857 mask16x32 {
17858 val: _mm512_cmplt_epi16_mask(a.into(), b.into()),
17859 simd: token,
17860 }
17861 }
17862 );
17863 kernel(self, a, b)
17864 }
17865 #[inline(always)]
17866 fn simd_le_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
17867 crate::kernel!(
17868 #[inline(always)]
17869 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> mask16x32<Avx512> {
17870 mask16x32 {
17871 val: _mm512_cmple_epi16_mask(a.into(), b.into()),
17872 simd: token,
17873 }
17874 }
17875 );
17876 kernel(self, a, b)
17877 }
17878 #[inline(always)]
17879 fn zip_low_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17880 crate::kernel!(
17881 #[inline(always)]
17882 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17883 _mm512_permutex2var_epi16(
17884 a.into(),
17885 _mm512_set_epi16(
17886 47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6,
17887 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0,
17888 ),
17889 b.into(),
17890 )
17891 .simd_into(token)
17892 }
17893 );
17894 kernel(self, a, b)
17895 }
17896 #[inline(always)]
17897 fn zip_high_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17898 crate::kernel!(
17899 #[inline(always)]
17900 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17901 _mm512_permutex2var_epi16(
17902 a.into(),
17903 _mm512_set_epi16(
17904 63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54,
17905 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16,
17906 ),
17907 b.into(),
17908 )
17909 .simd_into(token)
17910 }
17911 );
17912 kernel(self, a, b)
17913 }
17914 #[inline(always)]
17915 fn unzip_low_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17916 crate::kernel!(
17917 #[inline(always)]
17918 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17919 _mm512_permutex2var_epi16(
17920 a.into(),
17921 _mm512_set_epi16(
17922 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26,
17923 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
17924 ),
17925 b.into(),
17926 )
17927 .simd_into(token)
17928 }
17929 );
17930 kernel(self, a, b)
17931 }
17932 #[inline(always)]
17933 fn unzip_high_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
17934 crate::kernel!(
17935 #[inline(always)]
17936 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
17937 _mm512_permutex2var_epi16(
17938 a.into(),
17939 _mm512_set_epi16(
17940 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, 27,
17941 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
17942 ),
17943 b.into(),
17944 )
17945 .simd_into(token)
17946 }
17947 );
17948 kernel(self, a, b)
17949 }
17950 #[inline(always)]
17951 fn interleave_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> (i16x32<Self>, i16x32<Self>) {
17952 crate::kernel!(
17953 #[inline(always)]
17954 fn kernel(
17955 token: Avx512,
17956 a: i16x32<Avx512>,
17957 b: i16x32<Avx512>,
17958 ) -> (i16x32<Avx512>, i16x32<Avx512>) {
17959 let a = a.into();
17960 let b = b.into();
17961 (
17962 _mm512_permutex2var_epi16(
17963 a,
17964 _mm512_set_epi16(
17965 47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7,
17966 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0,
17967 ),
17968 b,
17969 )
17970 .simd_into(token),
17971 _mm512_permutex2var_epi16(
17972 a,
17973 _mm512_set_epi16(
17974 63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23,
17975 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16,
17976 ),
17977 b,
17978 )
17979 .simd_into(token),
17980 )
17981 }
17982 );
17983 kernel(self, a, b)
17984 }
17985 #[inline(always)]
17986 fn deinterleave_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> (i16x32<Self>, i16x32<Self>) {
17987 crate::kernel!(
17988 #[inline(always)]
17989 fn kernel(
17990 token: Avx512,
17991 a: i16x32<Avx512>,
17992 b: i16x32<Avx512>,
17993 ) -> (i16x32<Avx512>, i16x32<Avx512>) {
17994 let a = a.into();
17995 let b = b.into();
17996 (
17997 _mm512_permutex2var_epi16(
17998 a,
17999 _mm512_set_epi16(
18000 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28,
18001 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
18002 ),
18003 b,
18004 )
18005 .simd_into(token),
18006 _mm512_permutex2var_epi16(
18007 a,
18008 _mm512_set_epi16(
18009 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29,
18010 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
18011 ),
18012 b,
18013 )
18014 .simd_into(token),
18015 )
18016 }
18017 );
18018 kernel(self, a, b)
18019 }
18020 #[inline(always)]
18021 fn select_i16x32(self, a: mask16x32<Self>, b: i16x32<Self>, c: i16x32<Self>) -> i16x32<Self> {
18022 crate::kernel!(
18023 #[inline(always)]
18024 fn kernel(
18025 token: Avx512,
18026 a: mask16x32<Avx512>,
18027 b: i16x32<Avx512>,
18028 c: i16x32<Avx512>,
18029 ) -> i16x32<Avx512> {
18030 _mm512_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token)
18031 }
18032 );
18033 kernel(self, a, b, c)
18034 }
18035 #[inline(always)]
18036 fn split_i16x32(self, a: i16x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
18037 crate::kernel!(
18038 #[inline(always)]
18039 fn kernel(token: Avx512, a: i16x32<Avx512>) -> (i16x16<Avx512>, i16x16<Avx512>) {
18040 (
18041 _mm512_castsi512_si256(a.into()).simd_into(token),
18042 _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
18043 )
18044 }
18045 );
18046 kernel(self, a)
18047 }
18048 #[inline(always)]
18049 fn neg_i16x32(self, a: i16x32<Self>) -> i16x32<Self> {
18050 crate::kernel!(
18051 #[inline(always)]
18052 fn kernel(token: Avx512, a: i16x32<Avx512>) -> i16x32<Avx512> {
18053 _mm512_sub_epi16(_mm512_setzero_si512(), a.into()).simd_into(token)
18054 }
18055 );
18056 kernel(self, a)
18057 }
18058 #[inline(always)]
18059 fn widen_i16x32(self, a: i16x32<Self>) -> (i32x16<Self>, i32x16<Self>) {
18060 crate::kernel!(
18061 #[inline(always)]
18062 fn kernel(token: Avx512, a: i16x32<Avx512>) -> (i32x16<Avx512>, i32x16<Avx512>) {
18063 let raw = a.into();
18064 (
18065 _mm512_cvtepi16_epi32(_mm512_castsi512_si256(raw)).simd_into(token),
18066 _mm512_cvtepi16_epi32(_mm512_extracti64x4_epi64::<1>(raw)).simd_into(token),
18067 )
18068 }
18069 );
18070 kernel(self, a)
18071 }
18072 #[inline(always)]
18073 fn widening_mul_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> (i32x16<Self>, i32x16<Self>) {
18074 crate::kernel!(
18075 #[inline(always)]
18076 fn kernel(
18077 token: Avx512,
18078 a: i16x32<Avx512>,
18079 b: i16x32<Avx512>,
18080 ) -> (i32x16<Avx512>, i32x16<Avx512>) {
18081 let a = a.into();
18082 let b = b.into();
18083 let low = _mm512_mullo_epi16(a, b);
18084 let high = _mm512_mulhi_epi16(a, b);
18085 let result_low = _mm512_permutex2var_epi16(
18086 low,
18087 _mm512_set_epi16(
18088 47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6,
18089 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0,
18090 ),
18091 high,
18092 );
18093 let result_high = _mm512_permutex2var_epi16(
18094 low,
18095 _mm512_set_epi16(
18096 63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54,
18097 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16,
18098 ),
18099 high,
18100 );
18101 (result_low.simd_into(token), result_high.simd_into(token))
18102 }
18103 );
18104 kernel(self, a, b)
18105 }
18106 #[inline(always)]
18107 fn widening_add_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> (i32x16<Self>, i32x16<Self>) {
18108 let (a0, a1) = self.widen_i16x32(a);
18109 let (b0, b1) = self.widen_i16x32(b);
18110 (self.add_i32x16(a0, b0), self.add_i32x16(a1, b1))
18111 }
18112 #[inline(always)]
18113 fn pairwise_widening_add_i16x32(self, a: i16x32<Self>) -> i32x16<Self> {
18114 crate::kernel!(
18115 #[inline(always)]
18116 fn kernel(token: Avx512, a: i16x32<Avx512>) -> i32x16<Avx512> {
18117 ({ _mm512_madd_epi16(a.into(), _mm512_set1_epi16(1)) }).simd_into(token)
18118 }
18119 );
18120 kernel(self, a)
18121 }
18122 #[inline(always)]
18123 fn pairwise_dot_product_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i32x16<Self> {
18124 crate::kernel!(
18125 #[inline(always)]
18126 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i32x16<Avx512> {
18127 _mm512_madd_epi16(a.into(), b.into()).simd_into(token)
18128 }
18129 );
18130 kernel(self, a, b)
18131 }
18132 #[inline(always)]
18133 fn narrow_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i8x64<Self> {
18134 crate::kernel!(
18135 #[inline(always)]
18136 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i8x64<Avx512> {
18137 let low = _mm512_cvtepi16_epi8(a.into());
18138 let high = _mm512_cvtepi16_epi8(b.into());
18139 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(low), high).simd_into(token)
18140 }
18141 );
18142 kernel(self, a, b)
18143 }
18144 #[inline(always)]
18145 fn saturating_narrow_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i8x64<Self> {
18146 crate::kernel!(
18147 #[inline(always)]
18148 fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i8x64<Avx512> {
18149 _mm512_permutexvar_epi64(
18150 _mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7),
18151 _mm512_packs_epi16(a.into(), b.into()),
18152 )
18153 .simd_into(token)
18154 }
18155 );
18156 kernel(self, a, b)
18157 }
18158 #[inline(always)]
18159 fn relaxed_narrow_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i8x64<Self> {
18160 debug_assert!(
18161 a.as_slice()
18162 .iter()
18163 .chain(b.as_slice())
18164 .all(|&value| { value >= i8::MIN as i16 && value <= i8::MAX as i16 }),
18165 "relaxed_narrow inputs must fit in the destination type",
18166 );
18167 self.saturating_narrow_i16x32(a, b)
18168 }
18169 #[inline(always)]
18170 fn splat_u16x32(self, val: u16) -> u16x32<Self> {
18171 crate::kernel!(
18172 #[inline(always)]
18173 fn kernel(token: Avx512, val: u16) -> u16x32<Avx512> {
18174 _mm512_set1_epi16(val.cast_signed()).simd_into(token)
18175 }
18176 );
18177 kernel(self, val)
18178 }
18179 #[inline(always)]
18180 fn reverse_u16x32(self, a: u16x32<Self>) -> u16x32<Self> {
18181 let indices: u8x64<Self> = [
18182 62, 63, 60, 61, 58, 59, 56, 57, 54, 55, 52, 53, 50, 51, 48, 49, 46, 47, 44, 45, 42, 43,
18183 40, 41, 38, 39, 36, 37, 34, 35, 32, 33, 30, 31, 28, 29, 26, 27, 24, 25, 22, 23, 20, 21,
18184 18, 19, 16, 17, 14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1,
18185 ]
18186 .simd_into(self);
18187 self.swizzle_dyn_u16x32(a, indices)
18188 }
18189 #[inline(always)]
18190 fn slide_u16x32<const SHIFT: usize>(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18191 crate::kernel!(
18192 #[inline(always)]
18193 fn kernel(
18194 token: Avx512,
18195 a: u16x32<Avx512>,
18196 b: u16x32<Avx512>,
18197 shift: usize,
18198 ) -> u16x32<Avx512> {
18199 if shift >= 32usize {
18200 return b;
18201 }
18202 let idx = _mm512_add_epi8(
18203 _mm512_set_epi8(
18204 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
18205 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
18206 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
18207 5, 4, 3, 2, 1, 0,
18208 ),
18209 _mm512_set1_epi8((shift * 2usize) as i8),
18210 );
18211 let result = _mm512_permutex2var_epi8(
18212 Bytes::to_bytes(a).val.0,
18213 idx,
18214 Bytes::to_bytes(b).val.0,
18215 );
18216 Bytes::from_bytes(u8x64 {
18217 val: crate::support::Aligned512(result),
18218 simd: token,
18219 })
18220 }
18221 );
18222 kernel(self, a, b, SHIFT)
18223 }
18224 #[inline(always)]
18225 fn slide_within_blocks_u16x32<const SHIFT: usize>(
18226 self,
18227 a: u16x32<Self>,
18228 b: u16x32<Self>,
18229 ) -> u16x32<Self> {
18230 if SHIFT == 0 {
18231 return a;
18232 }
18233 if SHIFT >= 8usize {
18234 return b;
18235 }
18236 let a = Bytes::to_bytes(a).val.0;
18237 let b = Bytes::to_bytes(b).val.0;
18238 let result = dyn_alignr_512(self, b, a, SHIFT * 2usize);
18239 Bytes::from_bytes(u8x64 {
18240 val: crate::support::Aligned512(result),
18241 simd: self,
18242 })
18243 }
18244 #[inline(always)]
18245 fn compress_u16x32(self, values: u16x32<Self>, mask: mask16x32<Self>) -> u16x32<Self> {
18246 crate::kernel!(
18247 #[inline(always)]
18248 fn kernel(
18249 token: Avx512,
18250 values: u16x32<Avx512>,
18251 mask: mask16x32<Avx512>,
18252 ) -> u16x32<Avx512> {
18253 _mm512_maskz_compress_epi16(u64::from((mask).val) as u32, values.into())
18254 .simd_into(token)
18255 }
18256 );
18257 kernel(self, values, mask)
18258 }
18259 #[inline(always)]
18260 fn compress_merge_u16x32(
18261 self,
18262 values: u16x32<Self>,
18263 mask: mask16x32<Self>,
18264 merge: u16x32<Self>,
18265 ) -> u16x32<Self> {
18266 crate::kernel!(
18267 #[inline(always)]
18268 fn kernel(
18269 token: Avx512,
18270 values: u16x32<Avx512>,
18271 mask: mask16x32<Avx512>,
18272 merge: u16x32<Avx512>,
18273 ) -> u16x32<Avx512> {
18274 _mm512_mask_compress_epi16(
18275 merge.into(),
18276 u64::from((mask).val) as u32,
18277 values.into(),
18278 )
18279 .simd_into(token)
18280 }
18281 );
18282 kernel(self, values, mask, merge)
18283 }
18284 #[inline(always)]
18285 fn expand_u16x32(self, values: u16x32<Self>, mask: mask16x32<Self>) -> u16x32<Self> {
18286 crate::kernel!(
18287 #[inline(always)]
18288 fn kernel(
18289 token: Avx512,
18290 values: u16x32<Avx512>,
18291 mask: mask16x32<Avx512>,
18292 ) -> u16x32<Avx512> {
18293 _mm512_maskz_expand_epi16(u64::from((mask).val) as u32, values.into())
18294 .simd_into(token)
18295 }
18296 );
18297 kernel(self, values, mask)
18298 }
18299 #[inline(always)]
18300 fn expand_merge_u16x32(
18301 self,
18302 values: u16x32<Self>,
18303 mask: mask16x32<Self>,
18304 merge: u16x32<Self>,
18305 ) -> u16x32<Self> {
18306 crate::kernel!(
18307 #[inline(always)]
18308 fn kernel(
18309 token: Avx512,
18310 values: u16x32<Avx512>,
18311 mask: mask16x32<Avx512>,
18312 merge: u16x32<Avx512>,
18313 ) -> u16x32<Avx512> {
18314 _mm512_mask_expand_epi16(merge.into(), u64::from((mask).val) as u32, values.into())
18315 .simd_into(token)
18316 }
18317 );
18318 kernel(self, values, mask, merge)
18319 }
18320 #[inline(always)]
18321 fn count_ones_u16x32(self, a: u16x32<Self>) -> u16x32<Self> {
18322 crate::kernel!(
18323 #[inline(always)]
18324 fn kernel(token: Avx512, a: u16x32<Avx512>) -> u16x32<Avx512> {
18325 _mm512_popcnt_epi16(a.into()).simd_into(token)
18326 }
18327 );
18328 kernel(self, a)
18329 }
18330 #[inline(always)]
18331 fn count_zeros_u16x32(self, a: u16x32<Self>) -> u16x32<Self> {
18332 self.count_ones_u16x32(self.not_u16x32(a))
18333 }
18334 #[inline(always)]
18335 fn add_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18336 crate::kernel!(
18337 #[inline(always)]
18338 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18339 _mm512_add_epi16(a.into(), b.into()).simd_into(token)
18340 }
18341 );
18342 kernel(self, a, b)
18343 }
18344 #[inline(always)]
18345 fn saturating_add_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18346 crate::kernel!(
18347 #[inline(always)]
18348 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18349 _mm512_adds_epu16(a.into(), b.into()).simd_into(token)
18350 }
18351 );
18352 kernel(self, a, b)
18353 }
18354 #[inline(always)]
18355 fn sub_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18356 crate::kernel!(
18357 #[inline(always)]
18358 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18359 _mm512_sub_epi16(a.into(), b.into()).simd_into(token)
18360 }
18361 );
18362 kernel(self, a, b)
18363 }
18364 #[inline(always)]
18365 fn saturating_sub_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18366 crate::kernel!(
18367 #[inline(always)]
18368 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18369 _mm512_subs_epu16(a.into(), b.into()).simd_into(token)
18370 }
18371 );
18372 kernel(self, a, b)
18373 }
18374 #[inline(always)]
18375 fn mul_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18376 crate::kernel!(
18377 #[inline(always)]
18378 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18379 _mm512_mullo_epi16(a.into(), b.into()).simd_into(token)
18380 }
18381 );
18382 kernel(self, a, b)
18383 }
18384 #[inline(always)]
18385 fn and_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18386 crate::kernel!(
18387 #[inline(always)]
18388 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18389 _mm512_and_si512(a.into(), b.into()).simd_into(token)
18390 }
18391 );
18392 kernel(self, a, b)
18393 }
18394 #[inline(always)]
18395 fn or_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18396 crate::kernel!(
18397 #[inline(always)]
18398 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18399 _mm512_or_si512(a.into(), b.into()).simd_into(token)
18400 }
18401 );
18402 kernel(self, a, b)
18403 }
18404 #[inline(always)]
18405 fn xor_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18406 crate::kernel!(
18407 #[inline(always)]
18408 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18409 _mm512_xor_si512(a.into(), b.into()).simd_into(token)
18410 }
18411 );
18412 kernel(self, a, b)
18413 }
18414 #[inline(always)]
18415 fn not_u16x32(self, a: u16x32<Self>) -> u16x32<Self> {
18416 a ^ !0
18417 }
18418 #[inline(always)]
18419 fn shl_u16x32(self, a: u16x32<Self>, shift: u32) -> u16x32<Self> {
18420 crate::kernel!(
18421 #[inline(always)]
18422 fn kernel(token: Avx512, a: u16x32<Avx512>, shift: u32) -> u16x32<Avx512> {
18423 _mm512_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
18424 }
18425 );
18426 kernel(self, a, shift)
18427 }
18428 #[inline(always)]
18429 fn shlv_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18430 crate::kernel!(
18431 #[inline(always)]
18432 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18433 _mm512_sllv_epi16(a.into(), b.into()).simd_into(token)
18434 }
18435 );
18436 kernel(self, a, b)
18437 }
18438 #[inline(always)]
18439 fn shr_u16x32(self, a: u16x32<Self>, shift: u32) -> u16x32<Self> {
18440 crate::kernel!(
18441 #[inline(always)]
18442 fn kernel(token: Avx512, a: u16x32<Avx512>, shift: u32) -> u16x32<Avx512> {
18443 _mm512_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
18444 }
18445 );
18446 kernel(self, a, shift)
18447 }
18448 #[inline(always)]
18449 fn shrv_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18450 crate::kernel!(
18451 #[inline(always)]
18452 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18453 _mm512_srlv_epi16(a.into(), b.into()).simd_into(token)
18454 }
18455 );
18456 kernel(self, a, b)
18457 }
18458 #[inline(always)]
18459 fn max_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18460 crate::kernel!(
18461 #[inline(always)]
18462 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18463 _mm512_max_epu16(a.into(), b.into()).simd_into(token)
18464 }
18465 );
18466 kernel(self, a, b)
18467 }
18468 #[inline(always)]
18469 fn min_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18470 crate::kernel!(
18471 #[inline(always)]
18472 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18473 _mm512_min_epu16(a.into(), b.into()).simd_into(token)
18474 }
18475 );
18476 kernel(self, a, b)
18477 }
18478 #[inline(always)]
18479 fn clamp_u16x32(self, a: u16x32<Self>, b: u16x32<Self>, c: u16x32<Self>) -> u16x32<Self> {
18480 debug_assert!(
18481 b.simd_le(c).all_true(),
18482 "clamp requires min <= max for all elements"
18483 );
18484 a.max(b).min(c)
18485 }
18486 #[inline(always)]
18487 fn simd_eq_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
18488 crate::kernel!(
18489 #[inline(always)]
18490 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> mask16x32<Avx512> {
18491 mask16x32 {
18492 val: _mm512_cmpeq_epu16_mask(a.into(), b.into()),
18493 simd: token,
18494 }
18495 }
18496 );
18497 kernel(self, a, b)
18498 }
18499 #[inline(always)]
18500 fn simd_ne_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
18501 crate::kernel!(
18502 #[inline(always)]
18503 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> mask16x32<Avx512> {
18504 mask16x32 {
18505 val: _mm512_cmpneq_epu16_mask(a.into(), b.into()),
18506 simd: token,
18507 }
18508 }
18509 );
18510 kernel(self, a, b)
18511 }
18512 #[inline(always)]
18513 fn simd_lt_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
18514 crate::kernel!(
18515 #[inline(always)]
18516 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> mask16x32<Avx512> {
18517 mask16x32 {
18518 val: _mm512_cmplt_epu16_mask(a.into(), b.into()),
18519 simd: token,
18520 }
18521 }
18522 );
18523 kernel(self, a, b)
18524 }
18525 #[inline(always)]
18526 fn simd_le_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
18527 crate::kernel!(
18528 #[inline(always)]
18529 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> mask16x32<Avx512> {
18530 mask16x32 {
18531 val: _mm512_cmple_epu16_mask(a.into(), b.into()),
18532 simd: token,
18533 }
18534 }
18535 );
18536 kernel(self, a, b)
18537 }
18538 #[inline(always)]
18539 fn zip_low_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18540 crate::kernel!(
18541 #[inline(always)]
18542 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18543 _mm512_permutex2var_epi16(
18544 a.into(),
18545 _mm512_set_epi16(
18546 47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6,
18547 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0,
18548 ),
18549 b.into(),
18550 )
18551 .simd_into(token)
18552 }
18553 );
18554 kernel(self, a, b)
18555 }
18556 #[inline(always)]
18557 fn zip_high_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18558 crate::kernel!(
18559 #[inline(always)]
18560 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18561 _mm512_permutex2var_epi16(
18562 a.into(),
18563 _mm512_set_epi16(
18564 63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54,
18565 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16,
18566 ),
18567 b.into(),
18568 )
18569 .simd_into(token)
18570 }
18571 );
18572 kernel(self, a, b)
18573 }
18574 #[inline(always)]
18575 fn unzip_low_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18576 crate::kernel!(
18577 #[inline(always)]
18578 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18579 _mm512_permutex2var_epi16(
18580 a.into(),
18581 _mm512_set_epi16(
18582 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26,
18583 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
18584 ),
18585 b.into(),
18586 )
18587 .simd_into(token)
18588 }
18589 );
18590 kernel(self, a, b)
18591 }
18592 #[inline(always)]
18593 fn unzip_high_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
18594 crate::kernel!(
18595 #[inline(always)]
18596 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
18597 _mm512_permutex2var_epi16(
18598 a.into(),
18599 _mm512_set_epi16(
18600 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, 27,
18601 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
18602 ),
18603 b.into(),
18604 )
18605 .simd_into(token)
18606 }
18607 );
18608 kernel(self, a, b)
18609 }
18610 #[inline(always)]
18611 fn interleave_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> (u16x32<Self>, u16x32<Self>) {
18612 crate::kernel!(
18613 #[inline(always)]
18614 fn kernel(
18615 token: Avx512,
18616 a: u16x32<Avx512>,
18617 b: u16x32<Avx512>,
18618 ) -> (u16x32<Avx512>, u16x32<Avx512>) {
18619 let a = a.into();
18620 let b = b.into();
18621 (
18622 _mm512_permutex2var_epi16(
18623 a,
18624 _mm512_set_epi16(
18625 47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7,
18626 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0,
18627 ),
18628 b,
18629 )
18630 .simd_into(token),
18631 _mm512_permutex2var_epi16(
18632 a,
18633 _mm512_set_epi16(
18634 63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23,
18635 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16,
18636 ),
18637 b,
18638 )
18639 .simd_into(token),
18640 )
18641 }
18642 );
18643 kernel(self, a, b)
18644 }
18645 #[inline(always)]
18646 fn deinterleave_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> (u16x32<Self>, u16x32<Self>) {
18647 crate::kernel!(
18648 #[inline(always)]
18649 fn kernel(
18650 token: Avx512,
18651 a: u16x32<Avx512>,
18652 b: u16x32<Avx512>,
18653 ) -> (u16x32<Avx512>, u16x32<Avx512>) {
18654 let a = a.into();
18655 let b = b.into();
18656 (
18657 _mm512_permutex2var_epi16(
18658 a,
18659 _mm512_set_epi16(
18660 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28,
18661 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
18662 ),
18663 b,
18664 )
18665 .simd_into(token),
18666 _mm512_permutex2var_epi16(
18667 a,
18668 _mm512_set_epi16(
18669 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29,
18670 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
18671 ),
18672 b,
18673 )
18674 .simd_into(token),
18675 )
18676 }
18677 );
18678 kernel(self, a, b)
18679 }
18680 #[inline(always)]
18681 fn select_u16x32(self, a: mask16x32<Self>, b: u16x32<Self>, c: u16x32<Self>) -> u16x32<Self> {
18682 crate::kernel!(
18683 #[inline(always)]
18684 fn kernel(
18685 token: Avx512,
18686 a: mask16x32<Avx512>,
18687 b: u16x32<Avx512>,
18688 c: u16x32<Avx512>,
18689 ) -> u16x32<Avx512> {
18690 _mm512_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token)
18691 }
18692 );
18693 kernel(self, a, b, c)
18694 }
18695 #[inline(always)]
18696 fn split_u16x32(self, a: u16x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
18697 crate::kernel!(
18698 #[inline(always)]
18699 fn kernel(token: Avx512, a: u16x32<Avx512>) -> (u16x16<Avx512>, u16x16<Avx512>) {
18700 (
18701 _mm512_castsi512_si256(a.into()).simd_into(token),
18702 _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
18703 )
18704 }
18705 );
18706 kernel(self, a)
18707 }
18708 #[inline(always)]
18709 fn widen_u16x32(self, a: u16x32<Self>) -> (u32x16<Self>, u32x16<Self>) {
18710 crate::kernel!(
18711 #[inline(always)]
18712 fn kernel(token: Avx512, a: u16x32<Avx512>) -> (u32x16<Avx512>, u32x16<Avx512>) {
18713 let raw = a.into();
18714 (
18715 _mm512_cvtepu16_epi32(_mm512_castsi512_si256(raw)).simd_into(token),
18716 _mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64::<1>(raw)).simd_into(token),
18717 )
18718 }
18719 );
18720 kernel(self, a)
18721 }
18722 #[inline(always)]
18723 fn widening_mul_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> (u32x16<Self>, u32x16<Self>) {
18724 crate::kernel!(
18725 #[inline(always)]
18726 fn kernel(
18727 token: Avx512,
18728 a: u16x32<Avx512>,
18729 b: u16x32<Avx512>,
18730 ) -> (u32x16<Avx512>, u32x16<Avx512>) {
18731 let a = a.into();
18732 let b = b.into();
18733 let low = _mm512_mullo_epi16(a, b);
18734 let high = _mm512_mulhi_epu16(a, b);
18735 let result_low = _mm512_permutex2var_epi16(
18736 low,
18737 _mm512_set_epi16(
18738 47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6,
18739 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0,
18740 ),
18741 high,
18742 );
18743 let result_high = _mm512_permutex2var_epi16(
18744 low,
18745 _mm512_set_epi16(
18746 63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54,
18747 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16,
18748 ),
18749 high,
18750 );
18751 (result_low.simd_into(token), result_high.simd_into(token))
18752 }
18753 );
18754 kernel(self, a, b)
18755 }
18756 #[inline(always)]
18757 fn widening_add_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> (u32x16<Self>, u32x16<Self>) {
18758 let (a0, a1) = self.widen_u16x32(a);
18759 let (b0, b1) = self.widen_u16x32(b);
18760 (self.add_u32x16(a0, b0), self.add_u32x16(a1, b1))
18761 }
18762 #[inline(always)]
18763 fn pairwise_widening_add_u16x32(self, a: u16x32<Self>) -> u32x16<Self> {
18764 crate::kernel!(
18765 #[inline(always)]
18766 fn kernel(token: Avx512, a: u16x32<Avx512>) -> u32x16<Avx512> {
18767 ({
18768 let a = a.into();
18769 _mm512_add_epi32(
18770 _mm512_and_si512(a, _mm512_set1_epi32(65535)),
18771 _mm512_srli_epi32::<16>(a),
18772 )
18773 })
18774 .simd_into(token)
18775 }
18776 );
18777 kernel(self, a)
18778 }
18779 #[inline(always)]
18780 fn pairwise_dot_product_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u32x16<Self> {
18781 let (low, high) = self.widening_mul_u16x32(a, b);
18782 let (even, odd) = self.deinterleave_u32x16(low, high);
18783 self.add_u32x16(even, odd)
18784 }
18785 #[inline(always)]
18786 fn narrow_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u8x64<Self> {
18787 crate::kernel!(
18788 #[inline(always)]
18789 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u8x64<Avx512> {
18790 let low = _mm512_cvtepi16_epi8(a.into());
18791 let high = _mm512_cvtepi16_epi8(b.into());
18792 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(low), high).simd_into(token)
18793 }
18794 );
18795 kernel(self, a, b)
18796 }
18797 #[inline(always)]
18798 fn saturating_narrow_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u8x64<Self> {
18799 crate::kernel!(
18800 #[inline(always)]
18801 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u8x64<Avx512> {
18802 _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), {
18803 let max = _mm512_set1_epi16(i16::MAX);
18804 _mm512_packus_epi16(
18805 _mm512_min_epu16(a.into(), max),
18806 _mm512_min_epu16(b.into(), max),
18807 )
18808 })
18809 .simd_into(token)
18810 }
18811 );
18812 kernel(self, a, b)
18813 }
18814 #[inline(always)]
18815 fn relaxed_narrow_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u8x64<Self> {
18816 crate::kernel!(
18817 #[inline(always)]
18818 fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u8x64<Avx512> {
18819 debug_assert!(
18820 a.as_slice()
18821 .iter()
18822 .chain(b.as_slice())
18823 .all(|&value| { value >= u8::MIN as u16 && value <= u8::MAX as u16 }),
18824 "relaxed_narrow inputs must fit in the destination type",
18825 );
18826 _mm512_permutexvar_epi64(
18827 _mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7),
18828 _mm512_packus_epi16(a.into(), b.into()),
18829 )
18830 .simd_into(token)
18831 }
18832 );
18833 kernel(self, a, b)
18834 }
18835 #[inline(always)]
18836 fn splat_mask16x32(self, val: bool) -> mask16x32<Self> {
18837 mask16x32 {
18838 val: (if val { 4294967295u64 } else { 0 }) as _,
18839 simd: self,
18840 }
18841 }
18842 #[inline(always)]
18843 fn from_bitmask_mask16x32(self, bits: u64) -> mask16x32<Self> {
18844 mask16x32 {
18845 val: (bits & 4294967295u64) as _,
18846 simd: self,
18847 }
18848 }
18849 #[inline(always)]
18850 fn to_bitmask_mask16x32(self, a: mask16x32<Self>) -> u64 {
18851 u64::from((a).val) & 4294967295u64
18852 }
18853 #[inline(always)]
18854 fn from_vector_mask16x32(self, vector: i16x32<Self>) -> mask16x32<Self> {
18855 crate::kernel!(
18856 #[inline(always)]
18857 fn kernel(token: Avx512, vector: i16x32<Avx512>) -> mask16x32<Avx512> {
18858 _mm512_movepi16_mask(vector.into()).simd_into(token)
18859 }
18860 );
18861 kernel(self, vector)
18862 }
18863 #[inline(always)]
18864 fn to_vector_mask16x32(self, a: mask16x32<Self>) -> i16x32<Self> {
18865 crate::kernel!(
18866 #[inline(always)]
18867 fn kernel(token: Avx512, a: mask16x32<Avx512>) -> i16x32<Avx512> {
18868 _mm512_movm_epi16(a.val).simd_into(token)
18869 }
18870 );
18871 kernel(self, a)
18872 }
18873 #[inline(always)]
18874 fn set_mask16x32(self, a: &mut mask16x32<Self>, index: usize, value: bool) -> () {
18875 assert!(
18876 index < 32usize,
18877 "mask lane index {index} is out of bounds for {} lanes",
18878 32usize
18879 );
18880 let bit = 1u64 << index;
18881 let bits = u64::from((a).val);
18882 let bits = if value { bits | bit } else { bits & !bit };
18883 *a = mask16x32 {
18884 val: (bits) as _,
18885 simd: self,
18886 };
18887 }
18888 #[inline(always)]
18889 fn rotate_elements_left_mask16x32<const OFFSET: usize>(
18890 self,
18891 a: mask16x32<Self>,
18892 ) -> mask16x32<Self> {
18893 mask16x32 {
18894 val: (a.val.rotate_right((OFFSET % 32) as u32)) as _,
18895 simd: self,
18896 }
18897 }
18898 #[inline(always)]
18899 fn rotate_elements_right_mask16x32<const OFFSET: usize>(
18900 self,
18901 a: mask16x32<Self>,
18902 ) -> mask16x32<Self> {
18903 mask16x32 {
18904 val: (a.val.rotate_left((OFFSET % 32) as u32)) as _,
18905 simd: self,
18906 }
18907 }
18908 #[inline(always)]
18909 fn and_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask16x32<Self> {
18910 mask16x32 {
18911 val: ((u64::from((a).val) & u64::from((b).val)) & 4294967295u64) as _,
18912 simd: self,
18913 }
18914 }
18915 #[inline(always)]
18916 fn or_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask16x32<Self> {
18917 mask16x32 {
18918 val: ((u64::from((a).val) | u64::from((b).val)) & 4294967295u64) as _,
18919 simd: self,
18920 }
18921 }
18922 #[inline(always)]
18923 fn xor_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask16x32<Self> {
18924 mask16x32 {
18925 val: ((u64::from((a).val) ^ u64::from((b).val)) & 4294967295u64) as _,
18926 simd: self,
18927 }
18928 }
18929 #[inline(always)]
18930 fn not_mask16x32(self, a: mask16x32<Self>) -> mask16x32<Self> {
18931 mask16x32 {
18932 val: ((!u64::from((a).val)) & 4294967295u64) as _,
18933 simd: self,
18934 }
18935 }
18936 #[inline(always)]
18937 fn reverse_mask16x32(self, a: mask16x32<Self>) -> mask16x32<Self> {
18938 mask16x32 {
18939 val: (a.val.reverse_bits() >> 0usize) as _,
18940 simd: self,
18941 }
18942 }
18943 #[inline(always)]
18944 fn select_mask16x32(
18945 self,
18946 a: mask16x32<Self>,
18947 b: mask16x32<Self>,
18948 c: mask16x32<Self>,
18949 ) -> mask16x32<Self> {
18950 mask16x32 {
18951 val: (((u64::from((a).val) & u64::from((b).val))
18952 | ((!u64::from((a).val)) & u64::from((c).val)))
18953 & 4294967295u64) as _,
18954 simd: self,
18955 }
18956 }
18957 #[inline(always)]
18958 fn simd_eq_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask16x32<Self> {
18959 mask16x32 {
18960 val: (!u64::from(a.val ^ b.val) & 4294967295u64) as _,
18961 simd: self,
18962 }
18963 }
18964 #[inline(always)]
18965 fn simd_ne_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask16x32<Self> {
18966 mask16x32 {
18967 val: (u64::from(a.val ^ b.val) & 4294967295u64) as _,
18968 simd: self,
18969 }
18970 }
18971 #[inline(always)]
18972 fn any_true_mask16x32(self, a: mask16x32<Self>) -> bool {
18973 let bits = u64::from((a).val) & 4294967295u64;
18974 bits != 0
18975 }
18976 #[inline(always)]
18977 fn all_true_mask16x32(self, a: mask16x32<Self>) -> bool {
18978 let bits = u64::from((a).val) & 4294967295u64;
18979 bits == 4294967295u64
18980 }
18981 #[inline(always)]
18982 fn any_false_mask16x32(self, a: mask16x32<Self>) -> bool {
18983 let bits = u64::from((a).val) & 4294967295u64;
18984 bits != 4294967295u64
18985 }
18986 #[inline(always)]
18987 fn all_false_mask16x32(self, a: mask16x32<Self>) -> bool {
18988 let bits = u64::from((a).val) & 4294967295u64;
18989 bits == 0
18990 }
18991 #[inline(always)]
18992 fn split_mask16x32(self, a: mask16x32<Self>) -> (mask16x16<Self>, mask16x16<Self>) {
18993 let bits = u64::from(a.val);
18994 (
18995 mask16x16 {
18996 val: (bits & 65535u64) as _,
18997 simd: self,
18998 },
18999 mask16x16 {
19000 val: ((bits >> 16usize) & 65535u64) as _,
19001 simd: self,
19002 },
19003 )
19004 }
19005 #[inline(always)]
19006 fn widen_mask16x32(self, a: mask16x32<Self>) -> (mask32x16<Self>, mask32x16<Self>) {
19007 let bits = u64::from(a.val);
19008 (
19009 mask32x16 {
19010 val: (bits & 65535u64) as _,
19011 simd: self,
19012 },
19013 mask32x16 {
19014 val: ((bits >> 16usize) & 65535u64) as _,
19015 simd: self,
19016 },
19017 )
19018 }
19019 #[inline(always)]
19020 fn narrow_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask8x64<Self> {
19021 let low = u64::from(a.val) & 4294967295u64;
19022 let high = u64::from(b.val) & 4294967295u64;
19023 mask8x64 {
19024 val: low | (high << 32usize),
19025 simd: self,
19026 }
19027 }
19028 #[inline(always)]
19029 fn abs_i32x16(self, a: i32x16<Self>) -> i32x16<Self> {
19030 crate::kernel!(
19031 #[inline(always)]
19032 fn kernel(token: Avx512, a: i32x16<Avx512>) -> i32x16<Avx512> {
19033 _mm512_abs_epi32(a.into()).simd_into(token)
19034 }
19035 );
19036 kernel(self, a)
19037 }
19038 #[inline(always)]
19039 fn splat_i32x16(self, val: i32) -> i32x16<Self> {
19040 crate::kernel!(
19041 #[inline(always)]
19042 fn kernel(token: Avx512, val: i32) -> i32x16<Avx512> {
19043 _mm512_set1_epi32(val).simd_into(token)
19044 }
19045 );
19046 kernel(self, val)
19047 }
19048 #[inline(always)]
19049 fn reverse_i32x16(self, a: i32x16<Self>) -> i32x16<Self> {
19050 let indices: u8x64<Self> = [
19051 60, 61, 62, 63, 56, 57, 58, 59, 52, 53, 54, 55, 48, 49, 50, 51, 44, 45, 46, 47, 40, 41,
19052 42, 43, 36, 37, 38, 39, 32, 33, 34, 35, 28, 29, 30, 31, 24, 25, 26, 27, 20, 21, 22, 23,
19053 16, 17, 18, 19, 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3,
19054 ]
19055 .simd_into(self);
19056 self.swizzle_dyn_i32x16(a, indices)
19057 }
19058 #[inline(always)]
19059 fn slide_i32x16<const SHIFT: usize>(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19060 crate::kernel!(
19061 #[inline(always)]
19062 fn kernel(
19063 token: Avx512,
19064 a: i32x16<Avx512>,
19065 b: i32x16<Avx512>,
19066 shift: usize,
19067 ) -> i32x16<Avx512> {
19068 if shift >= 16usize {
19069 return b;
19070 }
19071 let idx = _mm512_add_epi8(
19072 _mm512_set_epi8(
19073 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
19074 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
19075 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
19076 5, 4, 3, 2, 1, 0,
19077 ),
19078 _mm512_set1_epi8((shift * 4usize) as i8),
19079 );
19080 let result = _mm512_permutex2var_epi8(
19081 Bytes::to_bytes(a).val.0,
19082 idx,
19083 Bytes::to_bytes(b).val.0,
19084 );
19085 Bytes::from_bytes(u8x64 {
19086 val: crate::support::Aligned512(result),
19087 simd: token,
19088 })
19089 }
19090 );
19091 kernel(self, a, b, SHIFT)
19092 }
19093 #[inline(always)]
19094 fn slide_within_blocks_i32x16<const SHIFT: usize>(
19095 self,
19096 a: i32x16<Self>,
19097 b: i32x16<Self>,
19098 ) -> i32x16<Self> {
19099 if SHIFT == 0 {
19100 return a;
19101 }
19102 if SHIFT >= 4usize {
19103 return b;
19104 }
19105 let a = Bytes::to_bytes(a).val.0;
19106 let b = Bytes::to_bytes(b).val.0;
19107 let result = dyn_alignr_512(self, b, a, SHIFT * 4usize);
19108 Bytes::from_bytes(u8x64 {
19109 val: crate::support::Aligned512(result),
19110 simd: self,
19111 })
19112 }
19113 #[inline(always)]
19114 fn compress_i32x16(self, values: i32x16<Self>, mask: mask32x16<Self>) -> i32x16<Self> {
19115 crate::kernel!(
19116 #[inline(always)]
19117 fn kernel(
19118 token: Avx512,
19119 values: i32x16<Avx512>,
19120 mask: mask32x16<Avx512>,
19121 ) -> i32x16<Avx512> {
19122 _mm512_maskz_compress_epi32(u64::from((mask).val) as u16, values.into())
19123 .simd_into(token)
19124 }
19125 );
19126 kernel(self, values, mask)
19127 }
19128 #[inline(always)]
19129 fn compress_merge_i32x16(
19130 self,
19131 values: i32x16<Self>,
19132 mask: mask32x16<Self>,
19133 merge: i32x16<Self>,
19134 ) -> i32x16<Self> {
19135 crate::kernel!(
19136 #[inline(always)]
19137 fn kernel(
19138 token: Avx512,
19139 values: i32x16<Avx512>,
19140 mask: mask32x16<Avx512>,
19141 merge: i32x16<Avx512>,
19142 ) -> i32x16<Avx512> {
19143 _mm512_mask_compress_epi32(
19144 merge.into(),
19145 u64::from((mask).val) as u16,
19146 values.into(),
19147 )
19148 .simd_into(token)
19149 }
19150 );
19151 kernel(self, values, mask, merge)
19152 }
19153 #[inline(always)]
19154 fn expand_i32x16(self, values: i32x16<Self>, mask: mask32x16<Self>) -> i32x16<Self> {
19155 crate::kernel!(
19156 #[inline(always)]
19157 fn kernel(
19158 token: Avx512,
19159 values: i32x16<Avx512>,
19160 mask: mask32x16<Avx512>,
19161 ) -> i32x16<Avx512> {
19162 _mm512_maskz_expand_epi32(u64::from((mask).val) as u16, values.into())
19163 .simd_into(token)
19164 }
19165 );
19166 kernel(self, values, mask)
19167 }
19168 #[inline(always)]
19169 fn expand_merge_i32x16(
19170 self,
19171 values: i32x16<Self>,
19172 mask: mask32x16<Self>,
19173 merge: i32x16<Self>,
19174 ) -> i32x16<Self> {
19175 crate::kernel!(
19176 #[inline(always)]
19177 fn kernel(
19178 token: Avx512,
19179 values: i32x16<Avx512>,
19180 mask: mask32x16<Avx512>,
19181 merge: i32x16<Avx512>,
19182 ) -> i32x16<Avx512> {
19183 _mm512_mask_expand_epi32(merge.into(), u64::from((mask).val) as u16, values.into())
19184 .simd_into(token)
19185 }
19186 );
19187 kernel(self, values, mask, merge)
19188 }
19189 #[inline(always)]
19190 fn count_ones_i32x16(self, a: i32x16<Self>) -> i32x16<Self> {
19191 crate::kernel!(
19192 #[inline(always)]
19193 fn kernel(token: Avx512, a: i32x16<Avx512>) -> i32x16<Avx512> {
19194 _mm512_popcnt_epi32(a.into()).simd_into(token)
19195 }
19196 );
19197 kernel(self, a)
19198 }
19199 #[inline(always)]
19200 fn count_zeros_i32x16(self, a: i32x16<Self>) -> i32x16<Self> {
19201 self.count_ones_i32x16(self.not_i32x16(a))
19202 }
19203 #[inline(always)]
19204 fn add_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19205 crate::kernel!(
19206 #[inline(always)]
19207 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19208 _mm512_add_epi32(a.into(), b.into()).simd_into(token)
19209 }
19210 );
19211 kernel(self, a, b)
19212 }
19213 #[inline(always)]
19214 fn saturating_add_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19215 crate::kernel!(
19216 #[inline(always)]
19217 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19218 let a = a.into();
19219 let b = b.into();
19220 let wrapped = _mm512_add_epi32(a, b);
19221 let overflow_bits = _mm512_ternarylogic_epi32::<0x42>(a, b, wrapped);
19222 let overflow_mask = _mm512_srai_epi32::<31>(overflow_bits);
19223 let direction =
19224 _mm512_add_epi32(_mm512_srli_epi32::<31>(a), _mm512_set1_epi32(i32::MAX));
19225 _mm512_ternarylogic_epi32::<0xca>(overflow_mask, direction, wrapped)
19226 .simd_into(token)
19227 }
19228 );
19229 kernel(self, a, b)
19230 }
19231 #[inline(always)]
19232 fn sub_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19233 crate::kernel!(
19234 #[inline(always)]
19235 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19236 _mm512_sub_epi32(a.into(), b.into()).simd_into(token)
19237 }
19238 );
19239 kernel(self, a, b)
19240 }
19241 #[inline(always)]
19242 fn saturating_sub_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19243 crate::kernel!(
19244 #[inline(always)]
19245 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19246 let a = a.into();
19247 let b = b.into();
19248 let wrapped = _mm512_sub_epi32(a, b);
19249 let overflow_bits = _mm512_ternarylogic_epi32::<0x18>(a, b, wrapped);
19250 let overflow_mask = _mm512_srai_epi32::<31>(overflow_bits);
19251 let direction =
19252 _mm512_add_epi32(_mm512_srli_epi32::<31>(a), _mm512_set1_epi32(i32::MAX));
19253 _mm512_ternarylogic_epi32::<0xca>(overflow_mask, direction, wrapped)
19254 .simd_into(token)
19255 }
19256 );
19257 kernel(self, a, b)
19258 }
19259 #[inline(always)]
19260 fn mul_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19261 crate::kernel!(
19262 #[inline(always)]
19263 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19264 _mm512_mullo_epi32(a.into(), b.into()).simd_into(token)
19265 }
19266 );
19267 kernel(self, a, b)
19268 }
19269 #[inline(always)]
19270 fn and_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19271 crate::kernel!(
19272 #[inline(always)]
19273 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19274 _mm512_and_si512(a.into(), b.into()).simd_into(token)
19275 }
19276 );
19277 kernel(self, a, b)
19278 }
19279 #[inline(always)]
19280 fn or_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19281 crate::kernel!(
19282 #[inline(always)]
19283 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19284 _mm512_or_si512(a.into(), b.into()).simd_into(token)
19285 }
19286 );
19287 kernel(self, a, b)
19288 }
19289 #[inline(always)]
19290 fn xor_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19291 crate::kernel!(
19292 #[inline(always)]
19293 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19294 _mm512_xor_si512(a.into(), b.into()).simd_into(token)
19295 }
19296 );
19297 kernel(self, a, b)
19298 }
19299 #[inline(always)]
19300 fn not_i32x16(self, a: i32x16<Self>) -> i32x16<Self> {
19301 a ^ !0
19302 }
19303 #[inline(always)]
19304 fn shl_i32x16(self, a: i32x16<Self>, shift: u32) -> i32x16<Self> {
19305 crate::kernel!(
19306 #[inline(always)]
19307 fn kernel(token: Avx512, a: i32x16<Avx512>, shift: u32) -> i32x16<Avx512> {
19308 _mm512_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
19309 }
19310 );
19311 kernel(self, a, shift)
19312 }
19313 #[inline(always)]
19314 fn shlv_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19315 crate::kernel!(
19316 #[inline(always)]
19317 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19318 _mm512_sllv_epi32(a.into(), b.into()).simd_into(token)
19319 }
19320 );
19321 kernel(self, a, b)
19322 }
19323 #[inline(always)]
19324 fn shr_i32x16(self, a: i32x16<Self>, shift: u32) -> i32x16<Self> {
19325 crate::kernel!(
19326 #[inline(always)]
19327 fn kernel(token: Avx512, a: i32x16<Avx512>, shift: u32) -> i32x16<Avx512> {
19328 _mm512_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
19329 }
19330 );
19331 kernel(self, a, shift)
19332 }
19333 #[inline(always)]
19334 fn shrv_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19335 crate::kernel!(
19336 #[inline(always)]
19337 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19338 _mm512_srav_epi32(a.into(), b.into()).simd_into(token)
19339 }
19340 );
19341 kernel(self, a, b)
19342 }
19343 #[inline(always)]
19344 fn max_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19345 crate::kernel!(
19346 #[inline(always)]
19347 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19348 _mm512_max_epi32(a.into(), b.into()).simd_into(token)
19349 }
19350 );
19351 kernel(self, a, b)
19352 }
19353 #[inline(always)]
19354 fn min_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19355 crate::kernel!(
19356 #[inline(always)]
19357 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19358 _mm512_min_epi32(a.into(), b.into()).simd_into(token)
19359 }
19360 );
19361 kernel(self, a, b)
19362 }
19363 #[inline(always)]
19364 fn clamp_i32x16(self, a: i32x16<Self>, b: i32x16<Self>, c: i32x16<Self>) -> i32x16<Self> {
19365 debug_assert!(
19366 b.simd_le(c).all_true(),
19367 "clamp requires min <= max for all elements"
19368 );
19369 a.max(b).min(c)
19370 }
19371 #[inline(always)]
19372 fn simd_eq_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
19373 crate::kernel!(
19374 #[inline(always)]
19375 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> mask32x16<Avx512> {
19376 mask32x16 {
19377 val: _mm512_cmpeq_epi32_mask(a.into(), b.into()),
19378 simd: token,
19379 }
19380 }
19381 );
19382 kernel(self, a, b)
19383 }
19384 #[inline(always)]
19385 fn simd_ne_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
19386 crate::kernel!(
19387 #[inline(always)]
19388 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> mask32x16<Avx512> {
19389 mask32x16 {
19390 val: _mm512_cmpneq_epi32_mask(a.into(), b.into()),
19391 simd: token,
19392 }
19393 }
19394 );
19395 kernel(self, a, b)
19396 }
19397 #[inline(always)]
19398 fn simd_lt_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
19399 crate::kernel!(
19400 #[inline(always)]
19401 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> mask32x16<Avx512> {
19402 mask32x16 {
19403 val: _mm512_cmplt_epi32_mask(a.into(), b.into()),
19404 simd: token,
19405 }
19406 }
19407 );
19408 kernel(self, a, b)
19409 }
19410 #[inline(always)]
19411 fn simd_le_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
19412 crate::kernel!(
19413 #[inline(always)]
19414 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> mask32x16<Avx512> {
19415 mask32x16 {
19416 val: _mm512_cmple_epi32_mask(a.into(), b.into()),
19417 simd: token,
19418 }
19419 }
19420 );
19421 kernel(self, a, b)
19422 }
19423 #[inline(always)]
19424 fn zip_low_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19425 crate::kernel!(
19426 #[inline(always)]
19427 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19428 _mm512_permutex2var_epi32(
19429 a.into(),
19430 _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
19431 b.into(),
19432 )
19433 .simd_into(token)
19434 }
19435 );
19436 kernel(self, a, b)
19437 }
19438 #[inline(always)]
19439 fn zip_high_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19440 crate::kernel!(
19441 #[inline(always)]
19442 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19443 _mm512_permutex2var_epi32(
19444 a.into(),
19445 _mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31),
19446 b.into(),
19447 )
19448 .simd_into(token)
19449 }
19450 );
19451 kernel(self, a, b)
19452 }
19453 #[inline(always)]
19454 fn unzip_low_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19455 crate::kernel!(
19456 #[inline(always)]
19457 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19458 _mm512_permutex2var_epi32(
19459 a.into(),
19460 _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
19461 b.into(),
19462 )
19463 .simd_into(token)
19464 }
19465 );
19466 kernel(self, a, b)
19467 }
19468 #[inline(always)]
19469 fn unzip_high_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
19470 crate::kernel!(
19471 #[inline(always)]
19472 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
19473 _mm512_permutex2var_epi32(
19474 a.into(),
19475 _mm512_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
19476 b.into(),
19477 )
19478 .simd_into(token)
19479 }
19480 );
19481 kernel(self, a, b)
19482 }
19483 #[inline(always)]
19484 fn interleave_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> (i32x16<Self>, i32x16<Self>) {
19485 crate::kernel!(
19486 #[inline(always)]
19487 fn kernel(
19488 token: Avx512,
19489 a: i32x16<Avx512>,
19490 b: i32x16<Avx512>,
19491 ) -> (i32x16<Avx512>, i32x16<Avx512>) {
19492 let a = a.into();
19493 let b = b.into();
19494 (
19495 _mm512_permutex2var_epi32(
19496 a,
19497 _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
19498 b,
19499 )
19500 .simd_into(token),
19501 _mm512_permutex2var_epi32(
19502 a,
19503 _mm512_setr_epi32(
19504 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31,
19505 ),
19506 b,
19507 )
19508 .simd_into(token),
19509 )
19510 }
19511 );
19512 kernel(self, a, b)
19513 }
19514 #[inline(always)]
19515 fn deinterleave_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> (i32x16<Self>, i32x16<Self>) {
19516 crate::kernel!(
19517 #[inline(always)]
19518 fn kernel(
19519 token: Avx512,
19520 a: i32x16<Avx512>,
19521 b: i32x16<Avx512>,
19522 ) -> (i32x16<Avx512>, i32x16<Avx512>) {
19523 let a = a.into();
19524 let b = b.into();
19525 (
19526 _mm512_permutex2var_epi32(
19527 a,
19528 _mm512_setr_epi32(
19529 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
19530 ),
19531 b,
19532 )
19533 .simd_into(token),
19534 _mm512_permutex2var_epi32(
19535 a,
19536 _mm512_setr_epi32(
19537 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31,
19538 ),
19539 b,
19540 )
19541 .simd_into(token),
19542 )
19543 }
19544 );
19545 kernel(self, a, b)
19546 }
19547 #[inline(always)]
19548 fn select_i32x16(self, a: mask32x16<Self>, b: i32x16<Self>, c: i32x16<Self>) -> i32x16<Self> {
19549 crate::kernel!(
19550 #[inline(always)]
19551 fn kernel(
19552 token: Avx512,
19553 a: mask32x16<Avx512>,
19554 b: i32x16<Avx512>,
19555 c: i32x16<Avx512>,
19556 ) -> i32x16<Avx512> {
19557 _mm512_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token)
19558 }
19559 );
19560 kernel(self, a, b, c)
19561 }
19562 #[inline(always)]
19563 fn split_i32x16(self, a: i32x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
19564 crate::kernel!(
19565 #[inline(always)]
19566 fn kernel(token: Avx512, a: i32x16<Avx512>) -> (i32x8<Avx512>, i32x8<Avx512>) {
19567 (
19568 _mm512_castsi512_si256(a.into()).simd_into(token),
19569 _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
19570 )
19571 }
19572 );
19573 kernel(self, a)
19574 }
19575 #[inline(always)]
19576 fn neg_i32x16(self, a: i32x16<Self>) -> i32x16<Self> {
19577 crate::kernel!(
19578 #[inline(always)]
19579 fn kernel(token: Avx512, a: i32x16<Avx512>) -> i32x16<Avx512> {
19580 _mm512_sub_epi32(_mm512_setzero_si512(), a.into()).simd_into(token)
19581 }
19582 );
19583 kernel(self, a)
19584 }
19585 #[inline(always)]
19586 fn widen_i32x16(self, a: i32x16<Self>) -> (i64x8<Self>, i64x8<Self>) {
19587 crate::kernel!(
19588 #[inline(always)]
19589 fn kernel(token: Avx512, a: i32x16<Avx512>) -> (i64x8<Avx512>, i64x8<Avx512>) {
19590 let raw = a.into();
19591 (
19592 _mm512_cvtepi32_epi64(_mm512_castsi512_si256(raw)).simd_into(token),
19593 _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64::<1>(raw)).simd_into(token),
19594 )
19595 }
19596 );
19597 kernel(self, a)
19598 }
19599 #[inline(always)]
19600 fn widening_mul_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> (i64x8<Self>, i64x8<Self>) {
19601 let (a0, a1) = self.widen_i32x16(a);
19602 let (b0, b1) = self.widen_i32x16(b);
19603 (self.mul_i64x8(a0, b0), self.mul_i64x8(a1, b1))
19604 }
19605 #[inline(always)]
19606 fn widening_add_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> (i64x8<Self>, i64x8<Self>) {
19607 let (a0, a1) = self.widen_i32x16(a);
19608 let (b0, b1) = self.widen_i32x16(b);
19609 (self.add_i64x8(a0, b0), self.add_i64x8(a1, b1))
19610 }
19611 #[inline(always)]
19612 fn pairwise_widening_add_i32x16(self, a: i32x16<Self>) -> i64x8<Self> {
19613 crate::kernel!(
19614 #[inline(always)]
19615 fn kernel(token: Avx512, a: i32x16<Avx512>) -> i64x8<Avx512> {
19616 ({
19617 let y = _mm512_xor_si512(a.into(), _mm512_set1_epi32(i32::MIN));
19618 _mm512_add_epi64(
19619 _mm512_or_si512(y, _mm512_set1_epi64(-0x1_0000_0000)),
19620 _mm512_srli_epi64::<32>(y),
19621 )
19622 })
19623 .simd_into(token)
19624 }
19625 );
19626 kernel(self, a)
19627 }
19628 #[inline(always)]
19629 fn pairwise_dot_product_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i64x8<Self> {
19630 let (low, high) = self.widening_mul_i32x16(a, b);
19631 let (even, odd) = self.deinterleave_i64x8(low, high);
19632 self.add_i64x8(even, odd)
19633 }
19634 #[inline(always)]
19635 fn narrow_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i16x32<Self> {
19636 crate::kernel!(
19637 #[inline(always)]
19638 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i16x32<Avx512> {
19639 let low = _mm512_cvtepi32_epi16(a.into());
19640 let high = _mm512_cvtepi32_epi16(b.into());
19641 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(low), high).simd_into(token)
19642 }
19643 );
19644 kernel(self, a, b)
19645 }
19646 #[inline(always)]
19647 fn saturating_narrow_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i16x32<Self> {
19648 crate::kernel!(
19649 #[inline(always)]
19650 fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i16x32<Avx512> {
19651 _mm512_permutexvar_epi64(
19652 _mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7),
19653 _mm512_packs_epi32(a.into(), b.into()),
19654 )
19655 .simd_into(token)
19656 }
19657 );
19658 kernel(self, a, b)
19659 }
19660 #[inline(always)]
19661 fn relaxed_narrow_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i16x32<Self> {
19662 debug_assert!(
19663 a.as_slice()
19664 .iter()
19665 .chain(b.as_slice())
19666 .all(|&value| { value >= i16::MIN as i32 && value <= i16::MAX as i32 }),
19667 "relaxed_narrow inputs must fit in the destination type",
19668 );
19669 self.saturating_narrow_i32x16(a, b)
19670 }
19671 #[inline(always)]
19672 fn cvt_f32_i32x16(self, a: i32x16<Self>) -> f32x16<Self> {
19673 crate::kernel!(
19674 #[inline(always)]
19675 fn kernel(token: Avx512, a: i32x16<Avx512>) -> f32x16<Avx512> {
19676 _mm512_cvtepi32_ps(a.into()).simd_into(token)
19677 }
19678 );
19679 kernel(self, a)
19680 }
19681 #[inline(always)]
19682 fn splat_u32x16(self, val: u32) -> u32x16<Self> {
19683 crate::kernel!(
19684 #[inline(always)]
19685 fn kernel(token: Avx512, val: u32) -> u32x16<Avx512> {
19686 _mm512_set1_epi32(val.cast_signed()).simd_into(token)
19687 }
19688 );
19689 kernel(self, val)
19690 }
19691 #[inline(always)]
19692 fn reverse_u32x16(self, a: u32x16<Self>) -> u32x16<Self> {
19693 let indices: u8x64<Self> = [
19694 60, 61, 62, 63, 56, 57, 58, 59, 52, 53, 54, 55, 48, 49, 50, 51, 44, 45, 46, 47, 40, 41,
19695 42, 43, 36, 37, 38, 39, 32, 33, 34, 35, 28, 29, 30, 31, 24, 25, 26, 27, 20, 21, 22, 23,
19696 16, 17, 18, 19, 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3,
19697 ]
19698 .simd_into(self);
19699 self.swizzle_dyn_u32x16(a, indices)
19700 }
19701 #[inline(always)]
19702 fn slide_u32x16<const SHIFT: usize>(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
19703 crate::kernel!(
19704 #[inline(always)]
19705 fn kernel(
19706 token: Avx512,
19707 a: u32x16<Avx512>,
19708 b: u32x16<Avx512>,
19709 shift: usize,
19710 ) -> u32x16<Avx512> {
19711 if shift >= 16usize {
19712 return b;
19713 }
19714 let idx = _mm512_add_epi8(
19715 _mm512_set_epi8(
19716 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
19717 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
19718 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
19719 5, 4, 3, 2, 1, 0,
19720 ),
19721 _mm512_set1_epi8((shift * 4usize) as i8),
19722 );
19723 let result = _mm512_permutex2var_epi8(
19724 Bytes::to_bytes(a).val.0,
19725 idx,
19726 Bytes::to_bytes(b).val.0,
19727 );
19728 Bytes::from_bytes(u8x64 {
19729 val: crate::support::Aligned512(result),
19730 simd: token,
19731 })
19732 }
19733 );
19734 kernel(self, a, b, SHIFT)
19735 }
19736 #[inline(always)]
19737 fn slide_within_blocks_u32x16<const SHIFT: usize>(
19738 self,
19739 a: u32x16<Self>,
19740 b: u32x16<Self>,
19741 ) -> u32x16<Self> {
19742 if SHIFT == 0 {
19743 return a;
19744 }
19745 if SHIFT >= 4usize {
19746 return b;
19747 }
19748 let a = Bytes::to_bytes(a).val.0;
19749 let b = Bytes::to_bytes(b).val.0;
19750 let result = dyn_alignr_512(self, b, a, SHIFT * 4usize);
19751 Bytes::from_bytes(u8x64 {
19752 val: crate::support::Aligned512(result),
19753 simd: self,
19754 })
19755 }
19756 #[inline(always)]
19757 fn compress_u32x16(self, values: u32x16<Self>, mask: mask32x16<Self>) -> u32x16<Self> {
19758 crate::kernel!(
19759 #[inline(always)]
19760 fn kernel(
19761 token: Avx512,
19762 values: u32x16<Avx512>,
19763 mask: mask32x16<Avx512>,
19764 ) -> u32x16<Avx512> {
19765 _mm512_maskz_compress_epi32(u64::from((mask).val) as u16, values.into())
19766 .simd_into(token)
19767 }
19768 );
19769 kernel(self, values, mask)
19770 }
19771 #[inline(always)]
19772 fn compress_merge_u32x16(
19773 self,
19774 values: u32x16<Self>,
19775 mask: mask32x16<Self>,
19776 merge: u32x16<Self>,
19777 ) -> u32x16<Self> {
19778 crate::kernel!(
19779 #[inline(always)]
19780 fn kernel(
19781 token: Avx512,
19782 values: u32x16<Avx512>,
19783 mask: mask32x16<Avx512>,
19784 merge: u32x16<Avx512>,
19785 ) -> u32x16<Avx512> {
19786 _mm512_mask_compress_epi32(
19787 merge.into(),
19788 u64::from((mask).val) as u16,
19789 values.into(),
19790 )
19791 .simd_into(token)
19792 }
19793 );
19794 kernel(self, values, mask, merge)
19795 }
19796 #[inline(always)]
19797 fn expand_u32x16(self, values: u32x16<Self>, mask: mask32x16<Self>) -> u32x16<Self> {
19798 crate::kernel!(
19799 #[inline(always)]
19800 fn kernel(
19801 token: Avx512,
19802 values: u32x16<Avx512>,
19803 mask: mask32x16<Avx512>,
19804 ) -> u32x16<Avx512> {
19805 _mm512_maskz_expand_epi32(u64::from((mask).val) as u16, values.into())
19806 .simd_into(token)
19807 }
19808 );
19809 kernel(self, values, mask)
19810 }
19811 #[inline(always)]
19812 fn expand_merge_u32x16(
19813 self,
19814 values: u32x16<Self>,
19815 mask: mask32x16<Self>,
19816 merge: u32x16<Self>,
19817 ) -> u32x16<Self> {
19818 crate::kernel!(
19819 #[inline(always)]
19820 fn kernel(
19821 token: Avx512,
19822 values: u32x16<Avx512>,
19823 mask: mask32x16<Avx512>,
19824 merge: u32x16<Avx512>,
19825 ) -> u32x16<Avx512> {
19826 _mm512_mask_expand_epi32(merge.into(), u64::from((mask).val) as u16, values.into())
19827 .simd_into(token)
19828 }
19829 );
19830 kernel(self, values, mask, merge)
19831 }
19832 #[inline(always)]
19833 fn count_ones_u32x16(self, a: u32x16<Self>) -> u32x16<Self> {
19834 crate::kernel!(
19835 #[inline(always)]
19836 fn kernel(token: Avx512, a: u32x16<Avx512>) -> u32x16<Avx512> {
19837 _mm512_popcnt_epi32(a.into()).simd_into(token)
19838 }
19839 );
19840 kernel(self, a)
19841 }
19842 #[inline(always)]
19843 fn count_zeros_u32x16(self, a: u32x16<Self>) -> u32x16<Self> {
19844 self.count_ones_u32x16(self.not_u32x16(a))
19845 }
19846 #[inline(always)]
19847 fn add_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
19848 crate::kernel!(
19849 #[inline(always)]
19850 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
19851 _mm512_add_epi32(a.into(), b.into()).simd_into(token)
19852 }
19853 );
19854 kernel(self, a, b)
19855 }
19856 #[inline(always)]
19857 fn saturating_add_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
19858 crate::kernel!(
19859 #[inline(always)]
19860 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
19861 let a = a.into();
19862 let b = b.into();
19863 let threshold = _mm512_xor_si512(b, _mm512_set1_epi32(-1));
19864 _mm512_add_epi32(_mm512_min_epu32(a, threshold), b).simd_into(token)
19865 }
19866 );
19867 kernel(self, a, b)
19868 }
19869 #[inline(always)]
19870 fn sub_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
19871 crate::kernel!(
19872 #[inline(always)]
19873 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
19874 _mm512_sub_epi32(a.into(), b.into()).simd_into(token)
19875 }
19876 );
19877 kernel(self, a, b)
19878 }
19879 #[inline(always)]
19880 fn saturating_sub_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
19881 crate::kernel!(
19882 #[inline(always)]
19883 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
19884 let a = a.into();
19885 let b = b.into();
19886 _mm512_sub_epi32(_mm512_max_epu32(a, b), b).simd_into(token)
19887 }
19888 );
19889 kernel(self, a, b)
19890 }
19891 #[inline(always)]
19892 fn mul_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
19893 crate::kernel!(
19894 #[inline(always)]
19895 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
19896 _mm512_mullo_epi32(a.into(), b.into()).simd_into(token)
19897 }
19898 );
19899 kernel(self, a, b)
19900 }
19901 #[inline(always)]
19902 fn and_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
19903 crate::kernel!(
19904 #[inline(always)]
19905 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
19906 _mm512_and_si512(a.into(), b.into()).simd_into(token)
19907 }
19908 );
19909 kernel(self, a, b)
19910 }
19911 #[inline(always)]
19912 fn or_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
19913 crate::kernel!(
19914 #[inline(always)]
19915 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
19916 _mm512_or_si512(a.into(), b.into()).simd_into(token)
19917 }
19918 );
19919 kernel(self, a, b)
19920 }
19921 #[inline(always)]
19922 fn xor_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
19923 crate::kernel!(
19924 #[inline(always)]
19925 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
19926 _mm512_xor_si512(a.into(), b.into()).simd_into(token)
19927 }
19928 );
19929 kernel(self, a, b)
19930 }
19931 #[inline(always)]
19932 fn not_u32x16(self, a: u32x16<Self>) -> u32x16<Self> {
19933 a ^ !0
19934 }
19935 #[inline(always)]
19936 fn shl_u32x16(self, a: u32x16<Self>, shift: u32) -> u32x16<Self> {
19937 crate::kernel!(
19938 #[inline(always)]
19939 fn kernel(token: Avx512, a: u32x16<Avx512>, shift: u32) -> u32x16<Avx512> {
19940 _mm512_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
19941 }
19942 );
19943 kernel(self, a, shift)
19944 }
19945 #[inline(always)]
19946 fn shlv_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
19947 crate::kernel!(
19948 #[inline(always)]
19949 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
19950 _mm512_sllv_epi32(a.into(), b.into()).simd_into(token)
19951 }
19952 );
19953 kernel(self, a, b)
19954 }
19955 #[inline(always)]
19956 fn shr_u32x16(self, a: u32x16<Self>, shift: u32) -> u32x16<Self> {
19957 crate::kernel!(
19958 #[inline(always)]
19959 fn kernel(token: Avx512, a: u32x16<Avx512>, shift: u32) -> u32x16<Avx512> {
19960 _mm512_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
19961 }
19962 );
19963 kernel(self, a, shift)
19964 }
19965 #[inline(always)]
19966 fn shrv_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
19967 crate::kernel!(
19968 #[inline(always)]
19969 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
19970 _mm512_srlv_epi32(a.into(), b.into()).simd_into(token)
19971 }
19972 );
19973 kernel(self, a, b)
19974 }
19975 #[inline(always)]
19976 fn max_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
19977 crate::kernel!(
19978 #[inline(always)]
19979 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
19980 _mm512_max_epu32(a.into(), b.into()).simd_into(token)
19981 }
19982 );
19983 kernel(self, a, b)
19984 }
19985 #[inline(always)]
19986 fn min_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
19987 crate::kernel!(
19988 #[inline(always)]
19989 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
19990 _mm512_min_epu32(a.into(), b.into()).simd_into(token)
19991 }
19992 );
19993 kernel(self, a, b)
19994 }
19995 #[inline(always)]
19996 fn clamp_u32x16(self, a: u32x16<Self>, b: u32x16<Self>, c: u32x16<Self>) -> u32x16<Self> {
19997 debug_assert!(
19998 b.simd_le(c).all_true(),
19999 "clamp requires min <= max for all elements"
20000 );
20001 a.max(b).min(c)
20002 }
20003 #[inline(always)]
20004 fn simd_eq_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
20005 crate::kernel!(
20006 #[inline(always)]
20007 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> mask32x16<Avx512> {
20008 mask32x16 {
20009 val: _mm512_cmpeq_epu32_mask(a.into(), b.into()),
20010 simd: token,
20011 }
20012 }
20013 );
20014 kernel(self, a, b)
20015 }
20016 #[inline(always)]
20017 fn simd_ne_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
20018 crate::kernel!(
20019 #[inline(always)]
20020 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> mask32x16<Avx512> {
20021 mask32x16 {
20022 val: _mm512_cmpneq_epu32_mask(a.into(), b.into()),
20023 simd: token,
20024 }
20025 }
20026 );
20027 kernel(self, a, b)
20028 }
20029 #[inline(always)]
20030 fn simd_lt_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
20031 crate::kernel!(
20032 #[inline(always)]
20033 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> mask32x16<Avx512> {
20034 mask32x16 {
20035 val: _mm512_cmplt_epu32_mask(a.into(), b.into()),
20036 simd: token,
20037 }
20038 }
20039 );
20040 kernel(self, a, b)
20041 }
20042 #[inline(always)]
20043 fn simd_le_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
20044 crate::kernel!(
20045 #[inline(always)]
20046 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> mask32x16<Avx512> {
20047 mask32x16 {
20048 val: _mm512_cmple_epu32_mask(a.into(), b.into()),
20049 simd: token,
20050 }
20051 }
20052 );
20053 kernel(self, a, b)
20054 }
20055 #[inline(always)]
20056 fn zip_low_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
20057 crate::kernel!(
20058 #[inline(always)]
20059 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
20060 _mm512_permutex2var_epi32(
20061 a.into(),
20062 _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
20063 b.into(),
20064 )
20065 .simd_into(token)
20066 }
20067 );
20068 kernel(self, a, b)
20069 }
20070 #[inline(always)]
20071 fn zip_high_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
20072 crate::kernel!(
20073 #[inline(always)]
20074 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
20075 _mm512_permutex2var_epi32(
20076 a.into(),
20077 _mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31),
20078 b.into(),
20079 )
20080 .simd_into(token)
20081 }
20082 );
20083 kernel(self, a, b)
20084 }
20085 #[inline(always)]
20086 fn unzip_low_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
20087 crate::kernel!(
20088 #[inline(always)]
20089 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
20090 _mm512_permutex2var_epi32(
20091 a.into(),
20092 _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
20093 b.into(),
20094 )
20095 .simd_into(token)
20096 }
20097 );
20098 kernel(self, a, b)
20099 }
20100 #[inline(always)]
20101 fn unzip_high_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
20102 crate::kernel!(
20103 #[inline(always)]
20104 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
20105 _mm512_permutex2var_epi32(
20106 a.into(),
20107 _mm512_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
20108 b.into(),
20109 )
20110 .simd_into(token)
20111 }
20112 );
20113 kernel(self, a, b)
20114 }
20115 #[inline(always)]
20116 fn interleave_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> (u32x16<Self>, u32x16<Self>) {
20117 crate::kernel!(
20118 #[inline(always)]
20119 fn kernel(
20120 token: Avx512,
20121 a: u32x16<Avx512>,
20122 b: u32x16<Avx512>,
20123 ) -> (u32x16<Avx512>, u32x16<Avx512>) {
20124 let a = a.into();
20125 let b = b.into();
20126 (
20127 _mm512_permutex2var_epi32(
20128 a,
20129 _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
20130 b,
20131 )
20132 .simd_into(token),
20133 _mm512_permutex2var_epi32(
20134 a,
20135 _mm512_setr_epi32(
20136 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31,
20137 ),
20138 b,
20139 )
20140 .simd_into(token),
20141 )
20142 }
20143 );
20144 kernel(self, a, b)
20145 }
20146 #[inline(always)]
20147 fn deinterleave_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> (u32x16<Self>, u32x16<Self>) {
20148 crate::kernel!(
20149 #[inline(always)]
20150 fn kernel(
20151 token: Avx512,
20152 a: u32x16<Avx512>,
20153 b: u32x16<Avx512>,
20154 ) -> (u32x16<Avx512>, u32x16<Avx512>) {
20155 let a = a.into();
20156 let b = b.into();
20157 (
20158 _mm512_permutex2var_epi32(
20159 a,
20160 _mm512_setr_epi32(
20161 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
20162 ),
20163 b,
20164 )
20165 .simd_into(token),
20166 _mm512_permutex2var_epi32(
20167 a,
20168 _mm512_setr_epi32(
20169 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31,
20170 ),
20171 b,
20172 )
20173 .simd_into(token),
20174 )
20175 }
20176 );
20177 kernel(self, a, b)
20178 }
20179 #[inline(always)]
20180 fn select_u32x16(self, a: mask32x16<Self>, b: u32x16<Self>, c: u32x16<Self>) -> u32x16<Self> {
20181 crate::kernel!(
20182 #[inline(always)]
20183 fn kernel(
20184 token: Avx512,
20185 a: mask32x16<Avx512>,
20186 b: u32x16<Avx512>,
20187 c: u32x16<Avx512>,
20188 ) -> u32x16<Avx512> {
20189 _mm512_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token)
20190 }
20191 );
20192 kernel(self, a, b, c)
20193 }
20194 #[inline(always)]
20195 fn split_u32x16(self, a: u32x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
20196 crate::kernel!(
20197 #[inline(always)]
20198 fn kernel(token: Avx512, a: u32x16<Avx512>) -> (u32x8<Avx512>, u32x8<Avx512>) {
20199 (
20200 _mm512_castsi512_si256(a.into()).simd_into(token),
20201 _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
20202 )
20203 }
20204 );
20205 kernel(self, a)
20206 }
20207 #[inline(always)]
20208 fn widen_u32x16(self, a: u32x16<Self>) -> (u64x8<Self>, u64x8<Self>) {
20209 crate::kernel!(
20210 #[inline(always)]
20211 fn kernel(token: Avx512, a: u32x16<Avx512>) -> (u64x8<Avx512>, u64x8<Avx512>) {
20212 let raw = a.into();
20213 (
20214 _mm512_cvtepu32_epi64(_mm512_castsi512_si256(raw)).simd_into(token),
20215 _mm512_cvtepu32_epi64(_mm512_extracti64x4_epi64::<1>(raw)).simd_into(token),
20216 )
20217 }
20218 );
20219 kernel(self, a)
20220 }
20221 #[inline(always)]
20222 fn widening_mul_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> (u64x8<Self>, u64x8<Self>) {
20223 let (a0, a1) = self.widen_u32x16(a);
20224 let (b0, b1) = self.widen_u32x16(b);
20225 (self.mul_u64x8(a0, b0), self.mul_u64x8(a1, b1))
20226 }
20227 #[inline(always)]
20228 fn widening_add_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> (u64x8<Self>, u64x8<Self>) {
20229 let (a0, a1) = self.widen_u32x16(a);
20230 let (b0, b1) = self.widen_u32x16(b);
20231 (self.add_u64x8(a0, b0), self.add_u64x8(a1, b1))
20232 }
20233 #[inline(always)]
20234 fn pairwise_widening_add_u32x16(self, a: u32x16<Self>) -> u64x8<Self> {
20235 crate::kernel!(
20236 #[inline(always)]
20237 fn kernel(token: Avx512, a: u32x16<Avx512>) -> u64x8<Avx512> {
20238 ({
20239 let a = a.into();
20240 _mm512_add_epi64(
20241 _mm512_and_si512(a, _mm512_set1_epi64(4294967295)),
20242 _mm512_srli_epi64::<32>(a),
20243 )
20244 })
20245 .simd_into(token)
20246 }
20247 );
20248 kernel(self, a)
20249 }
20250 #[inline(always)]
20251 fn pairwise_dot_product_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u64x8<Self> {
20252 let (low, high) = self.widening_mul_u32x16(a, b);
20253 let (even, odd) = self.deinterleave_u64x8(low, high);
20254 self.add_u64x8(even, odd)
20255 }
20256 #[inline(always)]
20257 fn narrow_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u16x32<Self> {
20258 crate::kernel!(
20259 #[inline(always)]
20260 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u16x32<Avx512> {
20261 let low = _mm512_cvtepi32_epi16(a.into());
20262 let high = _mm512_cvtepi32_epi16(b.into());
20263 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(low), high).simd_into(token)
20264 }
20265 );
20266 kernel(self, a, b)
20267 }
20268 #[inline(always)]
20269 fn saturating_narrow_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u16x32<Self> {
20270 crate::kernel!(
20271 #[inline(always)]
20272 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u16x32<Avx512> {
20273 _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), {
20274 let max = _mm512_set1_epi32(i32::MAX);
20275 _mm512_packus_epi32(
20276 _mm512_min_epu32(a.into(), max),
20277 _mm512_min_epu32(b.into(), max),
20278 )
20279 })
20280 .simd_into(token)
20281 }
20282 );
20283 kernel(self, a, b)
20284 }
20285 #[inline(always)]
20286 fn relaxed_narrow_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u16x32<Self> {
20287 crate::kernel!(
20288 #[inline(always)]
20289 fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u16x32<Avx512> {
20290 debug_assert!(
20291 a.as_slice()
20292 .iter()
20293 .chain(b.as_slice())
20294 .all(|&value| { value >= u16::MIN as u32 && value <= u16::MAX as u32 }),
20295 "relaxed_narrow inputs must fit in the destination type",
20296 );
20297 _mm512_permutexvar_epi64(
20298 _mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7),
20299 _mm512_packus_epi32(a.into(), b.into()),
20300 )
20301 .simd_into(token)
20302 }
20303 );
20304 kernel(self, a, b)
20305 }
20306 #[inline(always)]
20307 fn cvt_f32_u32x16(self, a: u32x16<Self>) -> f32x16<Self> {
20308 crate::kernel!(
20309 #[inline(always)]
20310 fn kernel(token: Avx512, a: u32x16<Avx512>) -> f32x16<Avx512> {
20311 _mm512_cvtepu32_ps(a.into()).simd_into(token)
20312 }
20313 );
20314 kernel(self, a)
20315 }
20316 #[inline(always)]
20317 fn splat_mask32x16(self, val: bool) -> mask32x16<Self> {
20318 mask32x16 {
20319 val: (if val { 65535u64 } else { 0 }) as _,
20320 simd: self,
20321 }
20322 }
20323 #[inline(always)]
20324 fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16<Self> {
20325 mask32x16 {
20326 val: (bits & 65535u64) as _,
20327 simd: self,
20328 }
20329 }
20330 #[inline(always)]
20331 fn to_bitmask_mask32x16(self, a: mask32x16<Self>) -> u64 {
20332 u64::from((a).val) & 65535u64
20333 }
20334 #[inline(always)]
20335 fn from_vector_mask32x16(self, vector: i32x16<Self>) -> mask32x16<Self> {
20336 crate::kernel!(
20337 #[inline(always)]
20338 fn kernel(token: Avx512, vector: i32x16<Avx512>) -> mask32x16<Avx512> {
20339 _mm512_movepi32_mask(vector.into()).simd_into(token)
20340 }
20341 );
20342 kernel(self, vector)
20343 }
20344 #[inline(always)]
20345 fn to_vector_mask32x16(self, a: mask32x16<Self>) -> i32x16<Self> {
20346 crate::kernel!(
20347 #[inline(always)]
20348 fn kernel(token: Avx512, a: mask32x16<Avx512>) -> i32x16<Avx512> {
20349 _mm512_movm_epi32(a.val).simd_into(token)
20350 }
20351 );
20352 kernel(self, a)
20353 }
20354 #[inline(always)]
20355 fn set_mask32x16(self, a: &mut mask32x16<Self>, index: usize, value: bool) -> () {
20356 assert!(
20357 index < 16usize,
20358 "mask lane index {index} is out of bounds for {} lanes",
20359 16usize
20360 );
20361 let bit = 1u64 << index;
20362 let bits = u64::from((a).val);
20363 let bits = if value { bits | bit } else { bits & !bit };
20364 *a = mask32x16 {
20365 val: (bits) as _,
20366 simd: self,
20367 };
20368 }
20369 #[inline(always)]
20370 fn rotate_elements_left_mask32x16<const OFFSET: usize>(
20371 self,
20372 a: mask32x16<Self>,
20373 ) -> mask32x16<Self> {
20374 mask32x16 {
20375 val: (a.val.rotate_right((OFFSET % 16) as u32)) as _,
20376 simd: self,
20377 }
20378 }
20379 #[inline(always)]
20380 fn rotate_elements_right_mask32x16<const OFFSET: usize>(
20381 self,
20382 a: mask32x16<Self>,
20383 ) -> mask32x16<Self> {
20384 mask32x16 {
20385 val: (a.val.rotate_left((OFFSET % 16) as u32)) as _,
20386 simd: self,
20387 }
20388 }
20389 #[inline(always)]
20390 fn and_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask32x16<Self> {
20391 mask32x16 {
20392 val: ((u64::from((a).val) & u64::from((b).val)) & 65535u64) as _,
20393 simd: self,
20394 }
20395 }
20396 #[inline(always)]
20397 fn or_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask32x16<Self> {
20398 mask32x16 {
20399 val: ((u64::from((a).val) | u64::from((b).val)) & 65535u64) as _,
20400 simd: self,
20401 }
20402 }
20403 #[inline(always)]
20404 fn xor_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask32x16<Self> {
20405 mask32x16 {
20406 val: ((u64::from((a).val) ^ u64::from((b).val)) & 65535u64) as _,
20407 simd: self,
20408 }
20409 }
20410 #[inline(always)]
20411 fn not_mask32x16(self, a: mask32x16<Self>) -> mask32x16<Self> {
20412 mask32x16 {
20413 val: ((!u64::from((a).val)) & 65535u64) as _,
20414 simd: self,
20415 }
20416 }
20417 #[inline(always)]
20418 fn reverse_mask32x16(self, a: mask32x16<Self>) -> mask32x16<Self> {
20419 mask32x16 {
20420 val: (a.val.reverse_bits() >> 0usize) as _,
20421 simd: self,
20422 }
20423 }
20424 #[inline(always)]
20425 fn select_mask32x16(
20426 self,
20427 a: mask32x16<Self>,
20428 b: mask32x16<Self>,
20429 c: mask32x16<Self>,
20430 ) -> mask32x16<Self> {
20431 mask32x16 {
20432 val: (((u64::from((a).val) & u64::from((b).val))
20433 | ((!u64::from((a).val)) & u64::from((c).val)))
20434 & 65535u64) as _,
20435 simd: self,
20436 }
20437 }
20438 #[inline(always)]
20439 fn simd_eq_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask32x16<Self> {
20440 mask32x16 {
20441 val: (!u64::from(a.val ^ b.val) & 65535u64) as _,
20442 simd: self,
20443 }
20444 }
20445 #[inline(always)]
20446 fn simd_ne_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask32x16<Self> {
20447 mask32x16 {
20448 val: (u64::from(a.val ^ b.val) & 65535u64) as _,
20449 simd: self,
20450 }
20451 }
20452 #[inline(always)]
20453 fn any_true_mask32x16(self, a: mask32x16<Self>) -> bool {
20454 let bits = u64::from((a).val) & 65535u64;
20455 bits != 0
20456 }
20457 #[inline(always)]
20458 fn all_true_mask32x16(self, a: mask32x16<Self>) -> bool {
20459 let bits = u64::from((a).val) & 65535u64;
20460 bits == 65535u64
20461 }
20462 #[inline(always)]
20463 fn any_false_mask32x16(self, a: mask32x16<Self>) -> bool {
20464 let bits = u64::from((a).val) & 65535u64;
20465 bits != 65535u64
20466 }
20467 #[inline(always)]
20468 fn all_false_mask32x16(self, a: mask32x16<Self>) -> bool {
20469 let bits = u64::from((a).val) & 65535u64;
20470 bits == 0
20471 }
20472 #[inline(always)]
20473 fn split_mask32x16(self, a: mask32x16<Self>) -> (mask32x8<Self>, mask32x8<Self>) {
20474 let bits = u64::from(a.val);
20475 (
20476 mask32x8 {
20477 val: (bits & 255u64) as _,
20478 simd: self,
20479 },
20480 mask32x8 {
20481 val: ((bits >> 8usize) & 255u64) as _,
20482 simd: self,
20483 },
20484 )
20485 }
20486 #[inline(always)]
20487 fn widen_mask32x16(self, a: mask32x16<Self>) -> (mask64x8<Self>, mask64x8<Self>) {
20488 let bits = u64::from(a.val);
20489 (
20490 mask64x8 {
20491 val: (bits & 255u64) as _,
20492 simd: self,
20493 },
20494 mask64x8 {
20495 val: ((bits >> 8usize) & 255u64) as _,
20496 simd: self,
20497 },
20498 )
20499 }
20500 #[inline(always)]
20501 fn narrow_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask16x32<Self> {
20502 let low = u64::from(a.val) & 65535u64;
20503 let high = u64::from(b.val) & 65535u64;
20504 mask16x32 {
20505 val: (low | (high << 16usize)) as _,
20506 simd: self,
20507 }
20508 }
20509 #[inline(always)]
20510 fn abs_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
20511 crate::kernel!(
20512 #[inline(always)]
20513 fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
20514 _mm512_andnot_pd(_mm512_set1_pd(-0.0), a.into()).simd_into(token)
20515 }
20516 );
20517 kernel(self, a)
20518 }
20519 #[inline(always)]
20520 fn splat_f64x8(self, val: f64) -> f64x8<Self> {
20521 crate::kernel!(
20522 #[inline(always)]
20523 fn kernel(token: Avx512, val: f64) -> f64x8<Avx512> {
20524 _mm512_set1_pd(val).simd_into(token)
20525 }
20526 );
20527 kernel(self, val)
20528 }
20529 #[inline(always)]
20530 fn reverse_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
20531 let indices: u8x64<Self> = [
20532 56, 57, 58, 59, 60, 61, 62, 63, 48, 49, 50, 51, 52, 53, 54, 55, 40, 41, 42, 43, 44, 45,
20533 46, 47, 32, 33, 34, 35, 36, 37, 38, 39, 24, 25, 26, 27, 28, 29, 30, 31, 16, 17, 18, 19,
20534 20, 21, 22, 23, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
20535 ]
20536 .simd_into(self);
20537 self.swizzle_dyn_f64x8(a, indices)
20538 }
20539 #[inline(always)]
20540 fn slide_f64x8<const SHIFT: usize>(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20541 crate::kernel!(
20542 #[inline(always)]
20543 fn kernel(
20544 token: Avx512,
20545 a: f64x8<Avx512>,
20546 b: f64x8<Avx512>,
20547 shift: usize,
20548 ) -> f64x8<Avx512> {
20549 if shift >= 8usize {
20550 return b;
20551 }
20552 let idx = _mm512_add_epi8(
20553 _mm512_set_epi8(
20554 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
20555 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
20556 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
20557 5, 4, 3, 2, 1, 0,
20558 ),
20559 _mm512_set1_epi8((shift * 8usize) as i8),
20560 );
20561 let result = _mm512_permutex2var_epi8(
20562 Bytes::to_bytes(a).val.0,
20563 idx,
20564 Bytes::to_bytes(b).val.0,
20565 );
20566 Bytes::from_bytes(u8x64 {
20567 val: crate::support::Aligned512(result),
20568 simd: token,
20569 })
20570 }
20571 );
20572 kernel(self, a, b, SHIFT)
20573 }
20574 #[inline(always)]
20575 fn slide_within_blocks_f64x8<const SHIFT: usize>(
20576 self,
20577 a: f64x8<Self>,
20578 b: f64x8<Self>,
20579 ) -> f64x8<Self> {
20580 if SHIFT == 0 {
20581 return a;
20582 }
20583 if SHIFT >= 2usize {
20584 return b;
20585 }
20586 let a = Bytes::to_bytes(a).val.0;
20587 let b = Bytes::to_bytes(b).val.0;
20588 let result = dyn_alignr_512(self, b, a, SHIFT * 8usize);
20589 Bytes::from_bytes(u8x64 {
20590 val: crate::support::Aligned512(result),
20591 simd: self,
20592 })
20593 }
20594 #[inline(always)]
20595 fn compress_f64x8(self, values: f64x8<Self>, mask: mask64x8<Self>) -> f64x8<Self> {
20596 crate::kernel!(
20597 #[inline(always)]
20598 fn kernel(
20599 token: Avx512,
20600 values: f64x8<Avx512>,
20601 mask: mask64x8<Avx512>,
20602 ) -> f64x8<Avx512> {
20603 _mm512_maskz_compress_pd(u64::from((mask).val) as u8, values.into())
20604 .simd_into(token)
20605 }
20606 );
20607 kernel(self, values, mask)
20608 }
20609 #[inline(always)]
20610 fn compress_merge_f64x8(
20611 self,
20612 values: f64x8<Self>,
20613 mask: mask64x8<Self>,
20614 merge: f64x8<Self>,
20615 ) -> f64x8<Self> {
20616 crate::kernel!(
20617 #[inline(always)]
20618 fn kernel(
20619 token: Avx512,
20620 values: f64x8<Avx512>,
20621 mask: mask64x8<Avx512>,
20622 merge: f64x8<Avx512>,
20623 ) -> f64x8<Avx512> {
20624 _mm512_mask_compress_pd(merge.into(), u64::from((mask).val) as u8, values.into())
20625 .simd_into(token)
20626 }
20627 );
20628 kernel(self, values, mask, merge)
20629 }
20630 #[inline(always)]
20631 fn expand_f64x8(self, values: f64x8<Self>, mask: mask64x8<Self>) -> f64x8<Self> {
20632 crate::kernel!(
20633 #[inline(always)]
20634 fn kernel(
20635 token: Avx512,
20636 values: f64x8<Avx512>,
20637 mask: mask64x8<Avx512>,
20638 ) -> f64x8<Avx512> {
20639 _mm512_maskz_expand_pd(u64::from((mask).val) as u8, values.into()).simd_into(token)
20640 }
20641 );
20642 kernel(self, values, mask)
20643 }
20644 #[inline(always)]
20645 fn expand_merge_f64x8(
20646 self,
20647 values: f64x8<Self>,
20648 mask: mask64x8<Self>,
20649 merge: f64x8<Self>,
20650 ) -> f64x8<Self> {
20651 crate::kernel!(
20652 #[inline(always)]
20653 fn kernel(
20654 token: Avx512,
20655 values: f64x8<Avx512>,
20656 mask: mask64x8<Avx512>,
20657 merge: f64x8<Avx512>,
20658 ) -> f64x8<Avx512> {
20659 _mm512_mask_expand_pd(merge.into(), u64::from((mask).val) as u8, values.into())
20660 .simd_into(token)
20661 }
20662 );
20663 kernel(self, values, mask, merge)
20664 }
20665 #[inline(always)]
20666 fn neg_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
20667 crate::kernel!(
20668 #[inline(always)]
20669 fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
20670 _mm512_xor_pd(a.into(), _mm512_set1_pd(-0.0)).simd_into(token)
20671 }
20672 );
20673 kernel(self, a)
20674 }
20675 #[inline(always)]
20676 fn sqrt_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
20677 crate::kernel!(
20678 #[inline(always)]
20679 fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
20680 _mm512_sqrt_pd(a.into()).simd_into(token)
20681 }
20682 );
20683 kernel(self, a)
20684 }
20685 #[inline(always)]
20686 fn approximate_recip_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
20687 crate::kernel!(
20688 #[inline(always)]
20689 fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
20690 _mm512_rcp14_pd(a.into()).simd_into(token)
20691 }
20692 );
20693 kernel(self, a)
20694 }
20695 #[inline(always)]
20696 fn add_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20697 crate::kernel!(
20698 #[inline(always)]
20699 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
20700 _mm512_add_pd(a.into(), b.into()).simd_into(token)
20701 }
20702 );
20703 kernel(self, a, b)
20704 }
20705 #[inline(always)]
20706 fn sub_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20707 crate::kernel!(
20708 #[inline(always)]
20709 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
20710 _mm512_sub_pd(a.into(), b.into()).simd_into(token)
20711 }
20712 );
20713 kernel(self, a, b)
20714 }
20715 #[inline(always)]
20716 fn mul_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20717 crate::kernel!(
20718 #[inline(always)]
20719 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
20720 _mm512_mul_pd(a.into(), b.into()).simd_into(token)
20721 }
20722 );
20723 kernel(self, a, b)
20724 }
20725 #[inline(always)]
20726 fn div_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20727 crate::kernel!(
20728 #[inline(always)]
20729 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
20730 _mm512_div_pd(a.into(), b.into()).simd_into(token)
20731 }
20732 );
20733 kernel(self, a, b)
20734 }
20735 #[inline(always)]
20736 fn copysign_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20737 crate::kernel!(
20738 #[inline(always)]
20739 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
20740 let mask = _mm512_set1_pd(-0.0);
20741 _mm512_or_pd(
20742 _mm512_and_pd(mask, b.into()),
20743 _mm512_andnot_pd(mask, a.into()),
20744 )
20745 .simd_into(token)
20746 }
20747 );
20748 kernel(self, a, b)
20749 }
20750 #[inline(always)]
20751 fn max_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20752 crate::kernel!(
20753 #[inline(always)]
20754 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
20755 _mm512_max_pd(a.into(), b.into()).simd_into(token)
20756 }
20757 );
20758 kernel(self, a, b)
20759 }
20760 #[inline(always)]
20761 fn min_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20762 crate::kernel!(
20763 #[inline(always)]
20764 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
20765 _mm512_min_pd(a.into(), b.into()).simd_into(token)
20766 }
20767 );
20768 kernel(self, a, b)
20769 }
20770 #[inline(always)]
20771 fn max_precise_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20772 crate::kernel!(
20773 #[inline(always)]
20774 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
20775 _mm512_range_pd::<5i32>(a.into(), b.into()).simd_into(token)
20776 }
20777 );
20778 kernel(self, a, b)
20779 }
20780 #[inline(always)]
20781 fn min_precise_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20782 crate::kernel!(
20783 #[inline(always)]
20784 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
20785 _mm512_range_pd::<4i32>(a.into(), b.into()).simd_into(token)
20786 }
20787 );
20788 kernel(self, a, b)
20789 }
20790 #[inline(always)]
20791 fn clamp_f64x8(self, a: f64x8<Self>, b: f64x8<Self>, c: f64x8<Self>) -> f64x8<Self> {
20792 crate::kernel!(
20793 #[inline(always)]
20794 fn kernel(
20795 token: Avx512,
20796 a: f64x8<Avx512>,
20797 b: f64x8<Avx512>,
20798 c: f64x8<Avx512>,
20799 ) -> f64x8<Avx512> {
20800 debug_assert!(
20801 b.simd_le(c).all_true(),
20802 "clamp requires min <= max for all elements"
20803 );
20804 _mm512_min_pd(c.into(), _mm512_max_pd(b.into(), a.into())).simd_into(token)
20805 }
20806 );
20807 kernel(self, a, b, c)
20808 }
20809 #[inline(always)]
20810 fn simd_eq_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
20811 crate::kernel!(
20812 #[inline(always)]
20813 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> mask64x8<Avx512> {
20814 mask64x8 {
20815 val: _mm512_cmp_pd_mask::<0i32>(a.into(), b.into()),
20816 simd: token,
20817 }
20818 }
20819 );
20820 kernel(self, a, b)
20821 }
20822 #[inline(always)]
20823 fn simd_ne_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
20824 crate::kernel!(
20825 #[inline(always)]
20826 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> mask64x8<Avx512> {
20827 mask64x8 {
20828 val: _mm512_cmp_pd_mask::<4i32>(a.into(), b.into()),
20829 simd: token,
20830 }
20831 }
20832 );
20833 kernel(self, a, b)
20834 }
20835 #[inline(always)]
20836 fn simd_lt_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
20837 crate::kernel!(
20838 #[inline(always)]
20839 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> mask64x8<Avx512> {
20840 mask64x8 {
20841 val: _mm512_cmp_pd_mask::<17i32>(a.into(), b.into()),
20842 simd: token,
20843 }
20844 }
20845 );
20846 kernel(self, a, b)
20847 }
20848 #[inline(always)]
20849 fn simd_le_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
20850 crate::kernel!(
20851 #[inline(always)]
20852 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> mask64x8<Avx512> {
20853 mask64x8 {
20854 val: _mm512_cmp_pd_mask::<18i32>(a.into(), b.into()),
20855 simd: token,
20856 }
20857 }
20858 );
20859 kernel(self, a, b)
20860 }
20861 #[inline(always)]
20862 fn zip_low_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20863 crate::kernel!(
20864 #[inline(always)]
20865 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
20866 _mm512_permutex2var_pd(
20867 a.into(),
20868 _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11),
20869 b.into(),
20870 )
20871 .simd_into(token)
20872 }
20873 );
20874 kernel(self, a, b)
20875 }
20876 #[inline(always)]
20877 fn zip_high_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20878 crate::kernel!(
20879 #[inline(always)]
20880 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
20881 _mm512_permutex2var_pd(
20882 a.into(),
20883 _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15),
20884 b.into(),
20885 )
20886 .simd_into(token)
20887 }
20888 );
20889 kernel(self, a, b)
20890 }
20891 #[inline(always)]
20892 fn unzip_low_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20893 crate::kernel!(
20894 #[inline(always)]
20895 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
20896 _mm512_permutex2var_pd(
20897 a.into(),
20898 _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14),
20899 b.into(),
20900 )
20901 .simd_into(token)
20902 }
20903 );
20904 kernel(self, a, b)
20905 }
20906 #[inline(always)]
20907 fn unzip_high_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
20908 crate::kernel!(
20909 #[inline(always)]
20910 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
20911 _mm512_permutex2var_pd(
20912 a.into(),
20913 _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15),
20914 b.into(),
20915 )
20916 .simd_into(token)
20917 }
20918 );
20919 kernel(self, a, b)
20920 }
20921 #[inline(always)]
20922 fn interleave_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> (f64x8<Self>, f64x8<Self>) {
20923 crate::kernel!(
20924 #[inline(always)]
20925 fn kernel(
20926 token: Avx512,
20927 a: f64x8<Avx512>,
20928 b: f64x8<Avx512>,
20929 ) -> (f64x8<Avx512>, f64x8<Avx512>) {
20930 let a = a.into();
20931 let b = b.into();
20932 (
20933 _mm512_permutex2var_pd(a, _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11), b)
20934 .simd_into(token),
20935 _mm512_permutex2var_pd(a, _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15), b)
20936 .simd_into(token),
20937 )
20938 }
20939 );
20940 kernel(self, a, b)
20941 }
20942 #[inline(always)]
20943 fn deinterleave_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> (f64x8<Self>, f64x8<Self>) {
20944 crate::kernel!(
20945 #[inline(always)]
20946 fn kernel(
20947 token: Avx512,
20948 a: f64x8<Avx512>,
20949 b: f64x8<Avx512>,
20950 ) -> (f64x8<Avx512>, f64x8<Avx512>) {
20951 let a = a.into();
20952 let b = b.into();
20953 (
20954 _mm512_permutex2var_pd(a, _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14), b)
20955 .simd_into(token),
20956 _mm512_permutex2var_pd(a, _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15), b)
20957 .simd_into(token),
20958 )
20959 }
20960 );
20961 kernel(self, a, b)
20962 }
20963 #[inline(always)]
20964 fn mul_add_f64x8(self, a: f64x8<Self>, b: f64x8<Self>, c: f64x8<Self>) -> f64x8<Self> {
20965 crate::kernel!(
20966 #[inline(always)]
20967 fn kernel(
20968 token: Avx512,
20969 a: f64x8<Avx512>,
20970 b: f64x8<Avx512>,
20971 c: f64x8<Avx512>,
20972 ) -> f64x8<Avx512> {
20973 _mm512_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token)
20974 }
20975 );
20976 kernel(self, a, b, c)
20977 }
20978 #[inline(always)]
20979 fn mul_add_precise_f64x8(self, a: f64x8<Self>, b: f64x8<Self>, c: f64x8<Self>) -> f64x8<Self> {
20980 self.mul_add_f64x8(a, b, c)
20981 }
20982 #[inline(always)]
20983 fn mul_sub_f64x8(self, a: f64x8<Self>, b: f64x8<Self>, c: f64x8<Self>) -> f64x8<Self> {
20984 crate::kernel!(
20985 #[inline(always)]
20986 fn kernel(
20987 token: Avx512,
20988 a: f64x8<Avx512>,
20989 b: f64x8<Avx512>,
20990 c: f64x8<Avx512>,
20991 ) -> f64x8<Avx512> {
20992 _mm512_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token)
20993 }
20994 );
20995 kernel(self, a, b, c)
20996 }
20997 #[inline(always)]
20998 fn mul_sub_precise_f64x8(self, a: f64x8<Self>, b: f64x8<Self>, c: f64x8<Self>) -> f64x8<Self> {
20999 self.mul_sub_f64x8(a, b, c)
21000 }
21001 #[inline(always)]
21002 fn floor_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
21003 crate::kernel!(
21004 #[inline(always)]
21005 fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
21006 _mm512_roundscale_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
21007 .simd_into(token)
21008 }
21009 );
21010 kernel(self, a)
21011 }
21012 #[inline(always)]
21013 fn ceil_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
21014 crate::kernel!(
21015 #[inline(always)]
21016 fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
21017 _mm512_roundscale_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
21018 .simd_into(token)
21019 }
21020 );
21021 kernel(self, a)
21022 }
21023 #[inline(always)]
21024 fn round_ties_even_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
21025 crate::kernel!(
21026 #[inline(always)]
21027 fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
21028 _mm512_roundscale_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
21029 .simd_into(token)
21030 }
21031 );
21032 kernel(self, a)
21033 }
21034 #[inline(always)]
21035 fn round_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
21036 let bias = self.splat_f64x8(const { f64::next_down(0.5) });
21037 (a + self.copysign_f64x8(bias, a)).trunc()
21038 }
21039 #[inline(always)]
21040 fn fract_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
21041 a - self.trunc_f64x8(a)
21042 }
21043 #[inline(always)]
21044 fn trunc_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
21045 crate::kernel!(
21046 #[inline(always)]
21047 fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
21048 _mm512_roundscale_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
21049 .simd_into(token)
21050 }
21051 );
21052 kernel(self, a)
21053 }
21054 #[inline(always)]
21055 fn to_degrees_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
21056 a * const { f64::to_degrees(1.0) }
21057 }
21058 #[inline(always)]
21059 fn to_radians_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
21060 a * const { f64::to_radians(1.0) }
21061 }
21062 #[inline(always)]
21063 fn is_nan_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
21064 crate::kernel!(
21065 #[inline(always)]
21066 fn kernel(token: Avx512, a: f64x8<Avx512>) -> mask64x8<Avx512> {
21067 mask64x8 {
21068 val: _mm512_fpclass_pd_mask::<129i32>(a.into()),
21069 simd: token,
21070 }
21071 }
21072 );
21073 kernel(self, a)
21074 }
21075 #[inline(always)]
21076 fn is_infinite_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
21077 crate::kernel!(
21078 #[inline(always)]
21079 fn kernel(token: Avx512, a: f64x8<Avx512>) -> mask64x8<Avx512> {
21080 mask64x8 {
21081 val: _mm512_fpclass_pd_mask::<24i32>(a.into()),
21082 simd: token,
21083 }
21084 }
21085 );
21086 kernel(self, a)
21087 }
21088 #[inline(always)]
21089 fn is_finite_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
21090 crate::kernel!(
21091 #[inline(always)]
21092 fn kernel(token: Avx512, a: f64x8<Avx512>) -> mask64x8<Avx512> {
21093 !(mask64x8 {
21094 val: _mm512_fpclass_pd_mask::<153i32>(a.into()),
21095 simd: token,
21096 })
21097 }
21098 );
21099 kernel(self, a)
21100 }
21101 #[inline(always)]
21102 fn is_subnormal_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
21103 crate::kernel!(
21104 #[inline(always)]
21105 fn kernel(token: Avx512, a: f64x8<Avx512>) -> mask64x8<Avx512> {
21106 mask64x8 {
21107 val: _mm512_fpclass_pd_mask::<129i32>(_mm512_xor_pd(
21108 a.into(),
21109 _mm512_set1_pd(f64::INFINITY),
21110 )),
21111 simd: token,
21112 }
21113 }
21114 );
21115 kernel(self, a)
21116 }
21117 #[inline(always)]
21118 fn is_normal_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
21119 crate::kernel!(
21120 #[inline(always)]
21121 fn kernel(token: Avx512, a: f64x8<Avx512>) -> mask64x8<Avx512> {
21122 !(mask64x8 {
21123 val: _mm512_fpclass_pd_mask::<191i32>(a.into()),
21124 simd: token,
21125 })
21126 }
21127 );
21128 kernel(self, a)
21129 }
21130 #[inline(always)]
21131 fn is_sign_positive_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
21132 let i: i64x8<Self> = a.bitcast();
21133 (i & i64::MIN).simd_eq(0)
21134 }
21135 #[inline(always)]
21136 fn is_sign_negative_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
21137 !a.is_sign_positive()
21138 }
21139 #[inline(always)]
21140 fn select_f64x8(self, a: mask64x8<Self>, b: f64x8<Self>, c: f64x8<Self>) -> f64x8<Self> {
21141 crate::kernel!(
21142 #[inline(always)]
21143 fn kernel(
21144 token: Avx512,
21145 a: mask64x8<Avx512>,
21146 b: f64x8<Avx512>,
21147 c: f64x8<Avx512>,
21148 ) -> f64x8<Avx512> {
21149 _mm512_mask_blend_pd(a.val, c.into(), b.into()).simd_into(token)
21150 }
21151 );
21152 kernel(self, a, b, c)
21153 }
21154 #[inline(always)]
21155 fn split_f64x8(self, a: f64x8<Self>) -> (f64x4<Self>, f64x4<Self>) {
21156 crate::kernel!(
21157 #[inline(always)]
21158 fn kernel(token: Avx512, a: f64x8<Avx512>) -> (f64x4<Avx512>, f64x4<Avx512>) {
21159 (
21160 _mm512_castpd512_pd256(a.into()).simd_into(token),
21161 _mm512_extractf64x4_pd::<1>(a.into()).simd_into(token),
21162 )
21163 }
21164 );
21165 kernel(self, a)
21166 }
21167 #[inline(always)]
21168 fn narrow_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f32x16<Self> {
21169 crate::kernel!(
21170 #[inline(always)]
21171 fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f32x16<Avx512> {
21172 let low = _mm512_cvtpd_ps(a.into());
21173 let high = _mm512_cvtpd_ps(b.into());
21174 _mm512_insertf32x8::<1>(_mm512_castps256_ps512(low), high).simd_into(token)
21175 }
21176 );
21177 kernel(self, a, b)
21178 }
21179 #[inline(always)]
21180 fn saturating_narrow_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f32x16<Self> {
21181 self.narrow_f64x8(a, b)
21182 }
21183 #[inline(always)]
21184 fn relaxed_narrow_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f32x16<Self> {
21185 self.narrow_f64x8(a, b)
21186 }
21187 #[inline(always)]
21188 fn cvt_u64_f64x8(self, a: f64x8<Self>) -> u64x8<Self> {
21189 crate::kernel!(
21190 #[inline(always)]
21191 fn kernel(token: Avx512, a: f64x8<Avx512>) -> u64x8<Avx512> {
21192 _mm512_cvttpd_epu64(a.into()).simd_into(token)
21193 }
21194 );
21195 kernel(self, a)
21196 }
21197 #[inline(always)]
21198 fn cvt_u64_precise_f64x8(self, a: f64x8<Self>) -> u64x8<Self> {
21199 crate::kernel!(
21200 #[inline(always)]
21201 fn kernel(token: Avx512, a: f64x8<Avx512>) -> u64x8<Avx512> {
21202 let a = a.into();
21203 let positive = _mm512_cmp_pd_mask::<17i32>(_mm512_setzero_pd(), a);
21204 _mm512_maskz_cvttpd_epu64(positive, a).simd_into(token)
21205 }
21206 );
21207 kernel(self, a)
21208 }
21209 #[inline(always)]
21210 fn cvt_i64_f64x8(self, a: f64x8<Self>) -> i64x8<Self> {
21211 crate::kernel!(
21212 #[inline(always)]
21213 fn kernel(token: Avx512, a: f64x8<Avx512>) -> i64x8<Avx512> {
21214 _mm512_cvttpd_epi64(a.into()).simd_into(token)
21215 }
21216 );
21217 kernel(self, a)
21218 }
21219 #[inline(always)]
21220 fn cvt_i64_precise_f64x8(self, a: f64x8<Self>) -> i64x8<Self> {
21221 crate::kernel!(
21222 #[inline(always)]
21223 fn kernel(token: Avx512, a: f64x8<Avx512>) -> i64x8<Avx512> {
21224 let a = a.into();
21225 let in_range =
21226 _mm512_cmp_pd_mask::<17i32>(a, _mm512_set1_pd(9223372036854775808.0));
21227 let mut converted =
21228 _mm512_mask_cvttpd_epi64(_mm512_set1_epi64(i64::MAX), in_range, a);
21229 let is_not_nan = _mm512_cmp_pd_mask::<7i32>(a, a);
21230 converted = _mm512_mask_blend_epi64(is_not_nan, _mm512_setzero_si512(), converted);
21231 converted.simd_into(token)
21232 }
21233 );
21234 kernel(self, a)
21235 }
21236 #[inline(always)]
21237 fn abs_i64x8(self, a: i64x8<Self>) -> i64x8<Self> {
21238 crate::kernel!(
21239 #[inline(always)]
21240 fn kernel(token: Avx512, a: i64x8<Avx512>) -> i64x8<Avx512> {
21241 _mm512_abs_epi64(a.into()).simd_into(token)
21242 }
21243 );
21244 kernel(self, a)
21245 }
21246 #[inline(always)]
21247 fn splat_i64x8(self, val: i64) -> i64x8<Self> {
21248 crate::kernel!(
21249 #[inline(always)]
21250 fn kernel(token: Avx512, val: i64) -> i64x8<Avx512> {
21251 _mm512_set1_epi64(val).simd_into(token)
21252 }
21253 );
21254 kernel(self, val)
21255 }
21256 #[inline(always)]
21257 fn reverse_i64x8(self, a: i64x8<Self>) -> i64x8<Self> {
21258 let indices: u8x64<Self> = [
21259 56, 57, 58, 59, 60, 61, 62, 63, 48, 49, 50, 51, 52, 53, 54, 55, 40, 41, 42, 43, 44, 45,
21260 46, 47, 32, 33, 34, 35, 36, 37, 38, 39, 24, 25, 26, 27, 28, 29, 30, 31, 16, 17, 18, 19,
21261 20, 21, 22, 23, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
21262 ]
21263 .simd_into(self);
21264 self.swizzle_dyn_i64x8(a, indices)
21265 }
21266 #[inline(always)]
21267 fn slide_i64x8<const SHIFT: usize>(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21268 crate::kernel!(
21269 #[inline(always)]
21270 fn kernel(
21271 token: Avx512,
21272 a: i64x8<Avx512>,
21273 b: i64x8<Avx512>,
21274 shift: usize,
21275 ) -> i64x8<Avx512> {
21276 if shift >= 8usize {
21277 return b;
21278 }
21279 let idx = _mm512_add_epi8(
21280 _mm512_set_epi8(
21281 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
21282 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
21283 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
21284 5, 4, 3, 2, 1, 0,
21285 ),
21286 _mm512_set1_epi8((shift * 8usize) as i8),
21287 );
21288 let result = _mm512_permutex2var_epi8(
21289 Bytes::to_bytes(a).val.0,
21290 idx,
21291 Bytes::to_bytes(b).val.0,
21292 );
21293 Bytes::from_bytes(u8x64 {
21294 val: crate::support::Aligned512(result),
21295 simd: token,
21296 })
21297 }
21298 );
21299 kernel(self, a, b, SHIFT)
21300 }
21301 #[inline(always)]
21302 fn slide_within_blocks_i64x8<const SHIFT: usize>(
21303 self,
21304 a: i64x8<Self>,
21305 b: i64x8<Self>,
21306 ) -> i64x8<Self> {
21307 if SHIFT == 0 {
21308 return a;
21309 }
21310 if SHIFT >= 2usize {
21311 return b;
21312 }
21313 let a = Bytes::to_bytes(a).val.0;
21314 let b = Bytes::to_bytes(b).val.0;
21315 let result = dyn_alignr_512(self, b, a, SHIFT * 8usize);
21316 Bytes::from_bytes(u8x64 {
21317 val: crate::support::Aligned512(result),
21318 simd: self,
21319 })
21320 }
21321 #[inline(always)]
21322 fn compress_i64x8(self, values: i64x8<Self>, mask: mask64x8<Self>) -> i64x8<Self> {
21323 crate::kernel!(
21324 #[inline(always)]
21325 fn kernel(
21326 token: Avx512,
21327 values: i64x8<Avx512>,
21328 mask: mask64x8<Avx512>,
21329 ) -> i64x8<Avx512> {
21330 _mm512_maskz_compress_epi64(u64::from((mask).val) as u8, values.into())
21331 .simd_into(token)
21332 }
21333 );
21334 kernel(self, values, mask)
21335 }
21336 #[inline(always)]
21337 fn compress_merge_i64x8(
21338 self,
21339 values: i64x8<Self>,
21340 mask: mask64x8<Self>,
21341 merge: i64x8<Self>,
21342 ) -> i64x8<Self> {
21343 crate::kernel!(
21344 #[inline(always)]
21345 fn kernel(
21346 token: Avx512,
21347 values: i64x8<Avx512>,
21348 mask: mask64x8<Avx512>,
21349 merge: i64x8<Avx512>,
21350 ) -> i64x8<Avx512> {
21351 _mm512_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into())
21352 .simd_into(token)
21353 }
21354 );
21355 kernel(self, values, mask, merge)
21356 }
21357 #[inline(always)]
21358 fn expand_i64x8(self, values: i64x8<Self>, mask: mask64x8<Self>) -> i64x8<Self> {
21359 crate::kernel!(
21360 #[inline(always)]
21361 fn kernel(
21362 token: Avx512,
21363 values: i64x8<Avx512>,
21364 mask: mask64x8<Avx512>,
21365 ) -> i64x8<Avx512> {
21366 _mm512_maskz_expand_epi64(u64::from((mask).val) as u8, values.into())
21367 .simd_into(token)
21368 }
21369 );
21370 kernel(self, values, mask)
21371 }
21372 #[inline(always)]
21373 fn expand_merge_i64x8(
21374 self,
21375 values: i64x8<Self>,
21376 mask: mask64x8<Self>,
21377 merge: i64x8<Self>,
21378 ) -> i64x8<Self> {
21379 crate::kernel!(
21380 #[inline(always)]
21381 fn kernel(
21382 token: Avx512,
21383 values: i64x8<Avx512>,
21384 mask: mask64x8<Avx512>,
21385 merge: i64x8<Avx512>,
21386 ) -> i64x8<Avx512> {
21387 _mm512_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into())
21388 .simd_into(token)
21389 }
21390 );
21391 kernel(self, values, mask, merge)
21392 }
21393 #[inline(always)]
21394 fn count_ones_i64x8(self, a: i64x8<Self>) -> i64x8<Self> {
21395 crate::kernel!(
21396 #[inline(always)]
21397 fn kernel(token: Avx512, a: i64x8<Avx512>) -> i64x8<Avx512> {
21398 _mm512_popcnt_epi64(a.into()).simd_into(token)
21399 }
21400 );
21401 kernel(self, a)
21402 }
21403 #[inline(always)]
21404 fn count_zeros_i64x8(self, a: i64x8<Self>) -> i64x8<Self> {
21405 self.count_ones_i64x8(self.not_i64x8(a))
21406 }
21407 #[inline(always)]
21408 fn add_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21409 crate::kernel!(
21410 #[inline(always)]
21411 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21412 _mm512_add_epi64(a.into(), b.into()).simd_into(token)
21413 }
21414 );
21415 kernel(self, a, b)
21416 }
21417 #[inline(always)]
21418 fn saturating_add_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21419 crate::kernel!(
21420 #[inline(always)]
21421 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21422 let a = a.into();
21423 let b = b.into();
21424 let wrapped = _mm512_add_epi64(a, b);
21425 let overflow_bits = _mm512_ternarylogic_epi64::<0x42>(a, b, wrapped);
21426 let overflow_mask = _mm512_srai_epi64::<63>(overflow_bits);
21427 let direction =
21428 _mm512_add_epi64(_mm512_srli_epi64::<63>(a), _mm512_set1_epi64(i64::MAX));
21429 _mm512_ternarylogic_epi64::<0xca>(overflow_mask, direction, wrapped)
21430 .simd_into(token)
21431 }
21432 );
21433 kernel(self, a, b)
21434 }
21435 #[inline(always)]
21436 fn sub_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21437 crate::kernel!(
21438 #[inline(always)]
21439 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21440 _mm512_sub_epi64(a.into(), b.into()).simd_into(token)
21441 }
21442 );
21443 kernel(self, a, b)
21444 }
21445 #[inline(always)]
21446 fn saturating_sub_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21447 crate::kernel!(
21448 #[inline(always)]
21449 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21450 let a = a.into();
21451 let b = b.into();
21452 let wrapped = _mm512_sub_epi64(a, b);
21453 let overflow_bits = _mm512_ternarylogic_epi64::<0x18>(a, b, wrapped);
21454 let overflow_mask = _mm512_srai_epi64::<63>(overflow_bits);
21455 let direction =
21456 _mm512_add_epi64(_mm512_srli_epi64::<63>(a), _mm512_set1_epi64(i64::MAX));
21457 _mm512_ternarylogic_epi64::<0xca>(overflow_mask, direction, wrapped)
21458 .simd_into(token)
21459 }
21460 );
21461 kernel(self, a, b)
21462 }
21463 #[inline(always)]
21464 fn mul_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21465 crate::kernel!(
21466 #[inline(always)]
21467 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21468 _mm512_mullo_epi64(a.into(), b.into()).simd_into(token)
21469 }
21470 );
21471 kernel(self, a, b)
21472 }
21473 #[inline(always)]
21474 fn and_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21475 crate::kernel!(
21476 #[inline(always)]
21477 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21478 _mm512_and_si512(a.into(), b.into()).simd_into(token)
21479 }
21480 );
21481 kernel(self, a, b)
21482 }
21483 #[inline(always)]
21484 fn or_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21485 crate::kernel!(
21486 #[inline(always)]
21487 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21488 _mm512_or_si512(a.into(), b.into()).simd_into(token)
21489 }
21490 );
21491 kernel(self, a, b)
21492 }
21493 #[inline(always)]
21494 fn xor_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21495 crate::kernel!(
21496 #[inline(always)]
21497 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21498 _mm512_xor_si512(a.into(), b.into()).simd_into(token)
21499 }
21500 );
21501 kernel(self, a, b)
21502 }
21503 #[inline(always)]
21504 fn not_i64x8(self, a: i64x8<Self>) -> i64x8<Self> {
21505 a ^ !0
21506 }
21507 #[inline(always)]
21508 fn shl_i64x8(self, a: i64x8<Self>, shift: u32) -> i64x8<Self> {
21509 crate::kernel!(
21510 #[inline(always)]
21511 fn kernel(token: Avx512, a: i64x8<Avx512>, shift: u32) -> i64x8<Avx512> {
21512 _mm512_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
21513 }
21514 );
21515 kernel(self, a, shift)
21516 }
21517 #[inline(always)]
21518 fn shlv_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21519 crate::kernel!(
21520 #[inline(always)]
21521 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21522 _mm512_sllv_epi64(a.into(), b.into()).simd_into(token)
21523 }
21524 );
21525 kernel(self, a, b)
21526 }
21527 #[inline(always)]
21528 fn shr_i64x8(self, a: i64x8<Self>, shift: u32) -> i64x8<Self> {
21529 crate::kernel!(
21530 #[inline(always)]
21531 fn kernel(token: Avx512, a: i64x8<Avx512>, shift: u32) -> i64x8<Avx512> {
21532 _mm512_sra_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
21533 }
21534 );
21535 kernel(self, a, shift)
21536 }
21537 #[inline(always)]
21538 fn shrv_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21539 crate::kernel!(
21540 #[inline(always)]
21541 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21542 _mm512_srav_epi64(a.into(), b.into()).simd_into(token)
21543 }
21544 );
21545 kernel(self, a, b)
21546 }
21547 #[inline(always)]
21548 fn max_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21549 crate::kernel!(
21550 #[inline(always)]
21551 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21552 _mm512_max_epi64(a.into(), b.into()).simd_into(token)
21553 }
21554 );
21555 kernel(self, a, b)
21556 }
21557 #[inline(always)]
21558 fn min_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21559 crate::kernel!(
21560 #[inline(always)]
21561 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21562 _mm512_min_epi64(a.into(), b.into()).simd_into(token)
21563 }
21564 );
21565 kernel(self, a, b)
21566 }
21567 #[inline(always)]
21568 fn clamp_i64x8(self, a: i64x8<Self>, b: i64x8<Self>, c: i64x8<Self>) -> i64x8<Self> {
21569 debug_assert!(
21570 b.simd_le(c).all_true(),
21571 "clamp requires min <= max for all elements"
21572 );
21573 a.max(b).min(c)
21574 }
21575 #[inline(always)]
21576 fn simd_eq_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> mask64x8<Self> {
21577 crate::kernel!(
21578 #[inline(always)]
21579 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> mask64x8<Avx512> {
21580 mask64x8 {
21581 val: _mm512_cmpeq_epi64_mask(a.into(), b.into()),
21582 simd: token,
21583 }
21584 }
21585 );
21586 kernel(self, a, b)
21587 }
21588 #[inline(always)]
21589 fn simd_ne_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> mask64x8<Self> {
21590 crate::kernel!(
21591 #[inline(always)]
21592 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> mask64x8<Avx512> {
21593 mask64x8 {
21594 val: _mm512_cmpneq_epi64_mask(a.into(), b.into()),
21595 simd: token,
21596 }
21597 }
21598 );
21599 kernel(self, a, b)
21600 }
21601 #[inline(always)]
21602 fn simd_lt_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> mask64x8<Self> {
21603 crate::kernel!(
21604 #[inline(always)]
21605 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> mask64x8<Avx512> {
21606 mask64x8 {
21607 val: _mm512_cmplt_epi64_mask(a.into(), b.into()),
21608 simd: token,
21609 }
21610 }
21611 );
21612 kernel(self, a, b)
21613 }
21614 #[inline(always)]
21615 fn simd_le_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> mask64x8<Self> {
21616 crate::kernel!(
21617 #[inline(always)]
21618 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> mask64x8<Avx512> {
21619 mask64x8 {
21620 val: _mm512_cmple_epi64_mask(a.into(), b.into()),
21621 simd: token,
21622 }
21623 }
21624 );
21625 kernel(self, a, b)
21626 }
21627 #[inline(always)]
21628 fn zip_low_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21629 crate::kernel!(
21630 #[inline(always)]
21631 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21632 _mm512_permutex2var_epi64(
21633 a.into(),
21634 _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11),
21635 b.into(),
21636 )
21637 .simd_into(token)
21638 }
21639 );
21640 kernel(self, a, b)
21641 }
21642 #[inline(always)]
21643 fn zip_high_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21644 crate::kernel!(
21645 #[inline(always)]
21646 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21647 _mm512_permutex2var_epi64(
21648 a.into(),
21649 _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15),
21650 b.into(),
21651 )
21652 .simd_into(token)
21653 }
21654 );
21655 kernel(self, a, b)
21656 }
21657 #[inline(always)]
21658 fn unzip_low_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21659 crate::kernel!(
21660 #[inline(always)]
21661 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21662 _mm512_permutex2var_epi64(
21663 a.into(),
21664 _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14),
21665 b.into(),
21666 )
21667 .simd_into(token)
21668 }
21669 );
21670 kernel(self, a, b)
21671 }
21672 #[inline(always)]
21673 fn unzip_high_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
21674 crate::kernel!(
21675 #[inline(always)]
21676 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i64x8<Avx512> {
21677 _mm512_permutex2var_epi64(
21678 a.into(),
21679 _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15),
21680 b.into(),
21681 )
21682 .simd_into(token)
21683 }
21684 );
21685 kernel(self, a, b)
21686 }
21687 #[inline(always)]
21688 fn interleave_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> (i64x8<Self>, i64x8<Self>) {
21689 crate::kernel!(
21690 #[inline(always)]
21691 fn kernel(
21692 token: Avx512,
21693 a: i64x8<Avx512>,
21694 b: i64x8<Avx512>,
21695 ) -> (i64x8<Avx512>, i64x8<Avx512>) {
21696 let a = a.into();
21697 let b = b.into();
21698 (
21699 _mm512_permutex2var_epi64(a, _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11), b)
21700 .simd_into(token),
21701 _mm512_permutex2var_epi64(a, _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15), b)
21702 .simd_into(token),
21703 )
21704 }
21705 );
21706 kernel(self, a, b)
21707 }
21708 #[inline(always)]
21709 fn deinterleave_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> (i64x8<Self>, i64x8<Self>) {
21710 crate::kernel!(
21711 #[inline(always)]
21712 fn kernel(
21713 token: Avx512,
21714 a: i64x8<Avx512>,
21715 b: i64x8<Avx512>,
21716 ) -> (i64x8<Avx512>, i64x8<Avx512>) {
21717 let a = a.into();
21718 let b = b.into();
21719 (
21720 _mm512_permutex2var_epi64(a, _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14), b)
21721 .simd_into(token),
21722 _mm512_permutex2var_epi64(a, _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15), b)
21723 .simd_into(token),
21724 )
21725 }
21726 );
21727 kernel(self, a, b)
21728 }
21729 #[inline(always)]
21730 fn select_i64x8(self, a: mask64x8<Self>, b: i64x8<Self>, c: i64x8<Self>) -> i64x8<Self> {
21731 crate::kernel!(
21732 #[inline(always)]
21733 fn kernel(
21734 token: Avx512,
21735 a: mask64x8<Avx512>,
21736 b: i64x8<Avx512>,
21737 c: i64x8<Avx512>,
21738 ) -> i64x8<Avx512> {
21739 _mm512_mask_blend_epi64(a.val, c.into(), b.into()).simd_into(token)
21740 }
21741 );
21742 kernel(self, a, b, c)
21743 }
21744 #[inline(always)]
21745 fn split_i64x8(self, a: i64x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
21746 crate::kernel!(
21747 #[inline(always)]
21748 fn kernel(token: Avx512, a: i64x8<Avx512>) -> (i64x4<Avx512>, i64x4<Avx512>) {
21749 (
21750 _mm512_castsi512_si256(a.into()).simd_into(token),
21751 _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
21752 )
21753 }
21754 );
21755 kernel(self, a)
21756 }
21757 #[inline(always)]
21758 fn neg_i64x8(self, a: i64x8<Self>) -> i64x8<Self> {
21759 crate::kernel!(
21760 #[inline(always)]
21761 fn kernel(token: Avx512, a: i64x8<Avx512>) -> i64x8<Avx512> {
21762 _mm512_sub_epi64(_mm512_setzero_si512(), a.into()).simd_into(token)
21763 }
21764 );
21765 kernel(self, a)
21766 }
21767 #[inline(always)]
21768 fn narrow_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i32x16<Self> {
21769 crate::kernel!(
21770 #[inline(always)]
21771 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i32x16<Avx512> {
21772 let low = _mm512_cvtepi64_epi32(a.into());
21773 let high = _mm512_cvtepi64_epi32(b.into());
21774 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(low), high).simd_into(token)
21775 }
21776 );
21777 kernel(self, a, b)
21778 }
21779 #[inline(always)]
21780 fn saturating_narrow_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i32x16<Self> {
21781 crate::kernel!(
21782 #[inline(always)]
21783 fn kernel(token: Avx512, a: i64x8<Avx512>, b: i64x8<Avx512>) -> i32x16<Avx512> {
21784 let low = _mm512_cvtsepi64_epi32(a.into());
21785 let high = _mm512_cvtsepi64_epi32(b.into());
21786 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(low), high).simd_into(token)
21787 }
21788 );
21789 kernel(self, a, b)
21790 }
21791 #[inline(always)]
21792 fn relaxed_narrow_i64x8(self, a: i64x8<Self>, b: i64x8<Self>) -> i32x16<Self> {
21793 debug_assert!(
21794 a.as_slice()
21795 .iter()
21796 .chain(b.as_slice())
21797 .all(|&value| { value >= i32::MIN as i64 && value <= i32::MAX as i64 }),
21798 "relaxed_narrow inputs must fit in the destination type",
21799 );
21800 self.narrow_i64x8(a, b)
21801 }
21802 #[inline(always)]
21803 fn cvt_f64_i64x8(self, a: i64x8<Self>) -> f64x8<Self> {
21804 crate::kernel!(
21805 #[inline(always)]
21806 fn kernel(token: Avx512, a: i64x8<Avx512>) -> f64x8<Avx512> {
21807 _mm512_cvtepi64_pd(a.into()).simd_into(token)
21808 }
21809 );
21810 kernel(self, a)
21811 }
21812 #[inline(always)]
21813 fn splat_u64x8(self, val: u64) -> u64x8<Self> {
21814 crate::kernel!(
21815 #[inline(always)]
21816 fn kernel(token: Avx512, val: u64) -> u64x8<Avx512> {
21817 _mm512_set1_epi64(val.cast_signed()).simd_into(token)
21818 }
21819 );
21820 kernel(self, val)
21821 }
21822 #[inline(always)]
21823 fn reverse_u64x8(self, a: u64x8<Self>) -> u64x8<Self> {
21824 let indices: u8x64<Self> = [
21825 56, 57, 58, 59, 60, 61, 62, 63, 48, 49, 50, 51, 52, 53, 54, 55, 40, 41, 42, 43, 44, 45,
21826 46, 47, 32, 33, 34, 35, 36, 37, 38, 39, 24, 25, 26, 27, 28, 29, 30, 31, 16, 17, 18, 19,
21827 20, 21, 22, 23, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
21828 ]
21829 .simd_into(self);
21830 self.swizzle_dyn_u64x8(a, indices)
21831 }
21832 #[inline(always)]
21833 fn slide_u64x8<const SHIFT: usize>(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
21834 crate::kernel!(
21835 #[inline(always)]
21836 fn kernel(
21837 token: Avx512,
21838 a: u64x8<Avx512>,
21839 b: u64x8<Avx512>,
21840 shift: usize,
21841 ) -> u64x8<Avx512> {
21842 if shift >= 8usize {
21843 return b;
21844 }
21845 let idx = _mm512_add_epi8(
21846 _mm512_set_epi8(
21847 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
21848 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
21849 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
21850 5, 4, 3, 2, 1, 0,
21851 ),
21852 _mm512_set1_epi8((shift * 8usize) as i8),
21853 );
21854 let result = _mm512_permutex2var_epi8(
21855 Bytes::to_bytes(a).val.0,
21856 idx,
21857 Bytes::to_bytes(b).val.0,
21858 );
21859 Bytes::from_bytes(u8x64 {
21860 val: crate::support::Aligned512(result),
21861 simd: token,
21862 })
21863 }
21864 );
21865 kernel(self, a, b, SHIFT)
21866 }
21867 #[inline(always)]
21868 fn slide_within_blocks_u64x8<const SHIFT: usize>(
21869 self,
21870 a: u64x8<Self>,
21871 b: u64x8<Self>,
21872 ) -> u64x8<Self> {
21873 if SHIFT == 0 {
21874 return a;
21875 }
21876 if SHIFT >= 2usize {
21877 return b;
21878 }
21879 let a = Bytes::to_bytes(a).val.0;
21880 let b = Bytes::to_bytes(b).val.0;
21881 let result = dyn_alignr_512(self, b, a, SHIFT * 8usize);
21882 Bytes::from_bytes(u8x64 {
21883 val: crate::support::Aligned512(result),
21884 simd: self,
21885 })
21886 }
21887 #[inline(always)]
21888 fn compress_u64x8(self, values: u64x8<Self>, mask: mask64x8<Self>) -> u64x8<Self> {
21889 crate::kernel!(
21890 #[inline(always)]
21891 fn kernel(
21892 token: Avx512,
21893 values: u64x8<Avx512>,
21894 mask: mask64x8<Avx512>,
21895 ) -> u64x8<Avx512> {
21896 _mm512_maskz_compress_epi64(u64::from((mask).val) as u8, values.into())
21897 .simd_into(token)
21898 }
21899 );
21900 kernel(self, values, mask)
21901 }
21902 #[inline(always)]
21903 fn compress_merge_u64x8(
21904 self,
21905 values: u64x8<Self>,
21906 mask: mask64x8<Self>,
21907 merge: u64x8<Self>,
21908 ) -> u64x8<Self> {
21909 crate::kernel!(
21910 #[inline(always)]
21911 fn kernel(
21912 token: Avx512,
21913 values: u64x8<Avx512>,
21914 mask: mask64x8<Avx512>,
21915 merge: u64x8<Avx512>,
21916 ) -> u64x8<Avx512> {
21917 _mm512_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into())
21918 .simd_into(token)
21919 }
21920 );
21921 kernel(self, values, mask, merge)
21922 }
21923 #[inline(always)]
21924 fn expand_u64x8(self, values: u64x8<Self>, mask: mask64x8<Self>) -> u64x8<Self> {
21925 crate::kernel!(
21926 #[inline(always)]
21927 fn kernel(
21928 token: Avx512,
21929 values: u64x8<Avx512>,
21930 mask: mask64x8<Avx512>,
21931 ) -> u64x8<Avx512> {
21932 _mm512_maskz_expand_epi64(u64::from((mask).val) as u8, values.into())
21933 .simd_into(token)
21934 }
21935 );
21936 kernel(self, values, mask)
21937 }
21938 #[inline(always)]
21939 fn expand_merge_u64x8(
21940 self,
21941 values: u64x8<Self>,
21942 mask: mask64x8<Self>,
21943 merge: u64x8<Self>,
21944 ) -> u64x8<Self> {
21945 crate::kernel!(
21946 #[inline(always)]
21947 fn kernel(
21948 token: Avx512,
21949 values: u64x8<Avx512>,
21950 mask: mask64x8<Avx512>,
21951 merge: u64x8<Avx512>,
21952 ) -> u64x8<Avx512> {
21953 _mm512_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into())
21954 .simd_into(token)
21955 }
21956 );
21957 kernel(self, values, mask, merge)
21958 }
21959 #[inline(always)]
21960 fn count_ones_u64x8(self, a: u64x8<Self>) -> u64x8<Self> {
21961 crate::kernel!(
21962 #[inline(always)]
21963 fn kernel(token: Avx512, a: u64x8<Avx512>) -> u64x8<Avx512> {
21964 _mm512_popcnt_epi64(a.into()).simd_into(token)
21965 }
21966 );
21967 kernel(self, a)
21968 }
21969 #[inline(always)]
21970 fn count_zeros_u64x8(self, a: u64x8<Self>) -> u64x8<Self> {
21971 self.count_ones_u64x8(self.not_u64x8(a))
21972 }
21973 #[inline(always)]
21974 fn add_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
21975 crate::kernel!(
21976 #[inline(always)]
21977 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
21978 _mm512_add_epi64(a.into(), b.into()).simd_into(token)
21979 }
21980 );
21981 kernel(self, a, b)
21982 }
21983 #[inline(always)]
21984 fn saturating_add_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
21985 crate::kernel!(
21986 #[inline(always)]
21987 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
21988 let a = a.into();
21989 let b = b.into();
21990 let threshold = _mm512_xor_si512(b, _mm512_set1_epi64(-1));
21991 _mm512_add_epi64(_mm512_min_epu64(a, threshold), b).simd_into(token)
21992 }
21993 );
21994 kernel(self, a, b)
21995 }
21996 #[inline(always)]
21997 fn sub_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
21998 crate::kernel!(
21999 #[inline(always)]
22000 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22001 _mm512_sub_epi64(a.into(), b.into()).simd_into(token)
22002 }
22003 );
22004 kernel(self, a, b)
22005 }
22006 #[inline(always)]
22007 fn saturating_sub_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
22008 crate::kernel!(
22009 #[inline(always)]
22010 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22011 let a = a.into();
22012 let b = b.into();
22013 _mm512_sub_epi64(_mm512_max_epu64(a, b), b).simd_into(token)
22014 }
22015 );
22016 kernel(self, a, b)
22017 }
22018 #[inline(always)]
22019 fn mul_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
22020 crate::kernel!(
22021 #[inline(always)]
22022 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22023 _mm512_mullo_epi64(a.into(), b.into()).simd_into(token)
22024 }
22025 );
22026 kernel(self, a, b)
22027 }
22028 #[inline(always)]
22029 fn and_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
22030 crate::kernel!(
22031 #[inline(always)]
22032 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22033 _mm512_and_si512(a.into(), b.into()).simd_into(token)
22034 }
22035 );
22036 kernel(self, a, b)
22037 }
22038 #[inline(always)]
22039 fn or_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
22040 crate::kernel!(
22041 #[inline(always)]
22042 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22043 _mm512_or_si512(a.into(), b.into()).simd_into(token)
22044 }
22045 );
22046 kernel(self, a, b)
22047 }
22048 #[inline(always)]
22049 fn xor_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
22050 crate::kernel!(
22051 #[inline(always)]
22052 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22053 _mm512_xor_si512(a.into(), b.into()).simd_into(token)
22054 }
22055 );
22056 kernel(self, a, b)
22057 }
22058 #[inline(always)]
22059 fn not_u64x8(self, a: u64x8<Self>) -> u64x8<Self> {
22060 a ^ !0
22061 }
22062 #[inline(always)]
22063 fn shl_u64x8(self, a: u64x8<Self>, shift: u32) -> u64x8<Self> {
22064 crate::kernel!(
22065 #[inline(always)]
22066 fn kernel(token: Avx512, a: u64x8<Avx512>, shift: u32) -> u64x8<Avx512> {
22067 _mm512_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
22068 }
22069 );
22070 kernel(self, a, shift)
22071 }
22072 #[inline(always)]
22073 fn shlv_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
22074 crate::kernel!(
22075 #[inline(always)]
22076 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22077 _mm512_sllv_epi64(a.into(), b.into()).simd_into(token)
22078 }
22079 );
22080 kernel(self, a, b)
22081 }
22082 #[inline(always)]
22083 fn shr_u64x8(self, a: u64x8<Self>, shift: u32) -> u64x8<Self> {
22084 crate::kernel!(
22085 #[inline(always)]
22086 fn kernel(token: Avx512, a: u64x8<Avx512>, shift: u32) -> u64x8<Avx512> {
22087 _mm512_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
22088 }
22089 );
22090 kernel(self, a, shift)
22091 }
22092 #[inline(always)]
22093 fn shrv_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
22094 crate::kernel!(
22095 #[inline(always)]
22096 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22097 _mm512_srlv_epi64(a.into(), b.into()).simd_into(token)
22098 }
22099 );
22100 kernel(self, a, b)
22101 }
22102 #[inline(always)]
22103 fn max_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
22104 crate::kernel!(
22105 #[inline(always)]
22106 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22107 _mm512_max_epu64(a.into(), b.into()).simd_into(token)
22108 }
22109 );
22110 kernel(self, a, b)
22111 }
22112 #[inline(always)]
22113 fn min_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
22114 crate::kernel!(
22115 #[inline(always)]
22116 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22117 _mm512_min_epu64(a.into(), b.into()).simd_into(token)
22118 }
22119 );
22120 kernel(self, a, b)
22121 }
22122 #[inline(always)]
22123 fn clamp_u64x8(self, a: u64x8<Self>, b: u64x8<Self>, c: u64x8<Self>) -> u64x8<Self> {
22124 debug_assert!(
22125 b.simd_le(c).all_true(),
22126 "clamp requires min <= max for all elements"
22127 );
22128 a.max(b).min(c)
22129 }
22130 #[inline(always)]
22131 fn simd_eq_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> mask64x8<Self> {
22132 crate::kernel!(
22133 #[inline(always)]
22134 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> mask64x8<Avx512> {
22135 mask64x8 {
22136 val: _mm512_cmpeq_epu64_mask(a.into(), b.into()),
22137 simd: token,
22138 }
22139 }
22140 );
22141 kernel(self, a, b)
22142 }
22143 #[inline(always)]
22144 fn simd_ne_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> mask64x8<Self> {
22145 crate::kernel!(
22146 #[inline(always)]
22147 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> mask64x8<Avx512> {
22148 mask64x8 {
22149 val: _mm512_cmpneq_epu64_mask(a.into(), b.into()),
22150 simd: token,
22151 }
22152 }
22153 );
22154 kernel(self, a, b)
22155 }
22156 #[inline(always)]
22157 fn simd_lt_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> mask64x8<Self> {
22158 crate::kernel!(
22159 #[inline(always)]
22160 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> mask64x8<Avx512> {
22161 mask64x8 {
22162 val: _mm512_cmplt_epu64_mask(a.into(), b.into()),
22163 simd: token,
22164 }
22165 }
22166 );
22167 kernel(self, a, b)
22168 }
22169 #[inline(always)]
22170 fn simd_le_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> mask64x8<Self> {
22171 crate::kernel!(
22172 #[inline(always)]
22173 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> mask64x8<Avx512> {
22174 mask64x8 {
22175 val: _mm512_cmple_epu64_mask(a.into(), b.into()),
22176 simd: token,
22177 }
22178 }
22179 );
22180 kernel(self, a, b)
22181 }
22182 #[inline(always)]
22183 fn zip_low_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
22184 crate::kernel!(
22185 #[inline(always)]
22186 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22187 _mm512_permutex2var_epi64(
22188 a.into(),
22189 _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11),
22190 b.into(),
22191 )
22192 .simd_into(token)
22193 }
22194 );
22195 kernel(self, a, b)
22196 }
22197 #[inline(always)]
22198 fn zip_high_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
22199 crate::kernel!(
22200 #[inline(always)]
22201 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22202 _mm512_permutex2var_epi64(
22203 a.into(),
22204 _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15),
22205 b.into(),
22206 )
22207 .simd_into(token)
22208 }
22209 );
22210 kernel(self, a, b)
22211 }
22212 #[inline(always)]
22213 fn unzip_low_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
22214 crate::kernel!(
22215 #[inline(always)]
22216 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22217 _mm512_permutex2var_epi64(
22218 a.into(),
22219 _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14),
22220 b.into(),
22221 )
22222 .simd_into(token)
22223 }
22224 );
22225 kernel(self, a, b)
22226 }
22227 #[inline(always)]
22228 fn unzip_high_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
22229 crate::kernel!(
22230 #[inline(always)]
22231 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u64x8<Avx512> {
22232 _mm512_permutex2var_epi64(
22233 a.into(),
22234 _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15),
22235 b.into(),
22236 )
22237 .simd_into(token)
22238 }
22239 );
22240 kernel(self, a, b)
22241 }
22242 #[inline(always)]
22243 fn interleave_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> (u64x8<Self>, u64x8<Self>) {
22244 crate::kernel!(
22245 #[inline(always)]
22246 fn kernel(
22247 token: Avx512,
22248 a: u64x8<Avx512>,
22249 b: u64x8<Avx512>,
22250 ) -> (u64x8<Avx512>, u64x8<Avx512>) {
22251 let a = a.into();
22252 let b = b.into();
22253 (
22254 _mm512_permutex2var_epi64(a, _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11), b)
22255 .simd_into(token),
22256 _mm512_permutex2var_epi64(a, _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15), b)
22257 .simd_into(token),
22258 )
22259 }
22260 );
22261 kernel(self, a, b)
22262 }
22263 #[inline(always)]
22264 fn deinterleave_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> (u64x8<Self>, u64x8<Self>) {
22265 crate::kernel!(
22266 #[inline(always)]
22267 fn kernel(
22268 token: Avx512,
22269 a: u64x8<Avx512>,
22270 b: u64x8<Avx512>,
22271 ) -> (u64x8<Avx512>, u64x8<Avx512>) {
22272 let a = a.into();
22273 let b = b.into();
22274 (
22275 _mm512_permutex2var_epi64(a, _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14), b)
22276 .simd_into(token),
22277 _mm512_permutex2var_epi64(a, _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15), b)
22278 .simd_into(token),
22279 )
22280 }
22281 );
22282 kernel(self, a, b)
22283 }
22284 #[inline(always)]
22285 fn select_u64x8(self, a: mask64x8<Self>, b: u64x8<Self>, c: u64x8<Self>) -> u64x8<Self> {
22286 crate::kernel!(
22287 #[inline(always)]
22288 fn kernel(
22289 token: Avx512,
22290 a: mask64x8<Avx512>,
22291 b: u64x8<Avx512>,
22292 c: u64x8<Avx512>,
22293 ) -> u64x8<Avx512> {
22294 _mm512_mask_blend_epi64(a.val, c.into(), b.into()).simd_into(token)
22295 }
22296 );
22297 kernel(self, a, b, c)
22298 }
22299 #[inline(always)]
22300 fn split_u64x8(self, a: u64x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
22301 crate::kernel!(
22302 #[inline(always)]
22303 fn kernel(token: Avx512, a: u64x8<Avx512>) -> (u64x4<Avx512>, u64x4<Avx512>) {
22304 (
22305 _mm512_castsi512_si256(a.into()).simd_into(token),
22306 _mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
22307 )
22308 }
22309 );
22310 kernel(self, a)
22311 }
22312 #[inline(always)]
22313 fn narrow_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u32x16<Self> {
22314 crate::kernel!(
22315 #[inline(always)]
22316 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u32x16<Avx512> {
22317 let low = _mm512_cvtepi64_epi32(a.into());
22318 let high = _mm512_cvtepi64_epi32(b.into());
22319 _mm512_inserti64x4::<1>(_mm512_castsi256_si512(low), high).simd_into(token)
22320 }
22321 );
22322 kernel(self, a, b)
22323 }
22324 #[inline(always)]
22325 fn saturating_narrow_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u32x16<Self> {
22326 crate::kernel!(
22327 #[inline(always)]
22328 fn kernel(token: Avx512, a: u64x8<Avx512>, b: u64x8<Avx512>) -> u32x16<Avx512> {
22329 let max = _mm512_set1_epi64(u32::MAX as i64);
22330 let a = _mm512_min_epu64(a.into(), max);
22331 let b = _mm512_min_epu64(b.into(), max);
22332 let indices =
22333 _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30);
22334 _mm512_permutex2var_epi32(a, indices, b).simd_into(token)
22335 }
22336 );
22337 kernel(self, a, b)
22338 }
22339 #[inline(always)]
22340 fn relaxed_narrow_u64x8(self, a: u64x8<Self>, b: u64x8<Self>) -> u32x16<Self> {
22341 debug_assert!(
22342 a.as_slice()
22343 .iter()
22344 .chain(b.as_slice())
22345 .all(|&value| { value >= u32::MIN as u64 && value <= u32::MAX as u64 }),
22346 "relaxed_narrow inputs must fit in the destination type",
22347 );
22348 self.saturating_narrow_u64x8(a, b)
22349 }
22350 #[inline(always)]
22351 fn cvt_f64_u64x8(self, a: u64x8<Self>) -> f64x8<Self> {
22352 crate::kernel!(
22353 #[inline(always)]
22354 fn kernel(token: Avx512, a: u64x8<Avx512>) -> f64x8<Avx512> {
22355 _mm512_cvtepu64_pd(a.into()).simd_into(token)
22356 }
22357 );
22358 kernel(self, a)
22359 }
22360 #[inline(always)]
22361 fn splat_mask64x8(self, val: bool) -> mask64x8<Self> {
22362 mask64x8 {
22363 val: (if val { 255u64 } else { 0 }) as _,
22364 simd: self,
22365 }
22366 }
22367 #[inline(always)]
22368 fn from_bitmask_mask64x8(self, bits: u64) -> mask64x8<Self> {
22369 mask64x8 {
22370 val: (bits & 255u64) as _,
22371 simd: self,
22372 }
22373 }
22374 #[inline(always)]
22375 fn to_bitmask_mask64x8(self, a: mask64x8<Self>) -> u64 {
22376 u64::from((a).val) & 255u64
22377 }
22378 #[inline(always)]
22379 fn from_vector_mask64x8(self, vector: i64x8<Self>) -> mask64x8<Self> {
22380 crate::kernel!(
22381 #[inline(always)]
22382 fn kernel(token: Avx512, vector: i64x8<Avx512>) -> mask64x8<Avx512> {
22383 _mm512_movepi64_mask(vector.into()).simd_into(token)
22384 }
22385 );
22386 kernel(self, vector)
22387 }
22388 #[inline(always)]
22389 fn to_vector_mask64x8(self, a: mask64x8<Self>) -> i64x8<Self> {
22390 crate::kernel!(
22391 #[inline(always)]
22392 fn kernel(token: Avx512, a: mask64x8<Avx512>) -> i64x8<Avx512> {
22393 _mm512_movm_epi64(a.val).simd_into(token)
22394 }
22395 );
22396 kernel(self, a)
22397 }
22398 #[inline(always)]
22399 fn set_mask64x8(self, a: &mut mask64x8<Self>, index: usize, value: bool) -> () {
22400 assert!(
22401 index < 8usize,
22402 "mask lane index {index} is out of bounds for {} lanes",
22403 8usize
22404 );
22405 let bit = 1u64 << index;
22406 let bits = u64::from((a).val);
22407 let bits = if value { bits | bit } else { bits & !bit };
22408 *a = mask64x8 {
22409 val: (bits) as _,
22410 simd: self,
22411 };
22412 }
22413 #[inline(always)]
22414 fn rotate_elements_left_mask64x8<const OFFSET: usize>(
22415 self,
22416 a: mask64x8<Self>,
22417 ) -> mask64x8<Self> {
22418 mask64x8 {
22419 val: (a.val.rotate_right((OFFSET % 8) as u32)) as _,
22420 simd: self,
22421 }
22422 }
22423 #[inline(always)]
22424 fn rotate_elements_right_mask64x8<const OFFSET: usize>(
22425 self,
22426 a: mask64x8<Self>,
22427 ) -> mask64x8<Self> {
22428 mask64x8 {
22429 val: (a.val.rotate_left((OFFSET % 8) as u32)) as _,
22430 simd: self,
22431 }
22432 }
22433 #[inline(always)]
22434 fn and_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask64x8<Self> {
22435 mask64x8 {
22436 val: ((u64::from((a).val) & u64::from((b).val)) & 255u64) as _,
22437 simd: self,
22438 }
22439 }
22440 #[inline(always)]
22441 fn or_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask64x8<Self> {
22442 mask64x8 {
22443 val: ((u64::from((a).val) | u64::from((b).val)) & 255u64) as _,
22444 simd: self,
22445 }
22446 }
22447 #[inline(always)]
22448 fn xor_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask64x8<Self> {
22449 mask64x8 {
22450 val: ((u64::from((a).val) ^ u64::from((b).val)) & 255u64) as _,
22451 simd: self,
22452 }
22453 }
22454 #[inline(always)]
22455 fn not_mask64x8(self, a: mask64x8<Self>) -> mask64x8<Self> {
22456 mask64x8 {
22457 val: ((!u64::from((a).val)) & 255u64) as _,
22458 simd: self,
22459 }
22460 }
22461 #[inline(always)]
22462 fn reverse_mask64x8(self, a: mask64x8<Self>) -> mask64x8<Self> {
22463 mask64x8 {
22464 val: (a.val.reverse_bits() >> 0usize) as _,
22465 simd: self,
22466 }
22467 }
22468 #[inline(always)]
22469 fn select_mask64x8(
22470 self,
22471 a: mask64x8<Self>,
22472 b: mask64x8<Self>,
22473 c: mask64x8<Self>,
22474 ) -> mask64x8<Self> {
22475 mask64x8 {
22476 val: (((u64::from((a).val) & u64::from((b).val))
22477 | ((!u64::from((a).val)) & u64::from((c).val)))
22478 & 255u64) as _,
22479 simd: self,
22480 }
22481 }
22482 #[inline(always)]
22483 fn simd_eq_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask64x8<Self> {
22484 mask64x8 {
22485 val: (!u64::from(a.val ^ b.val) & 255u64) as _,
22486 simd: self,
22487 }
22488 }
22489 #[inline(always)]
22490 fn simd_ne_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask64x8<Self> {
22491 mask64x8 {
22492 val: (u64::from(a.val ^ b.val) & 255u64) as _,
22493 simd: self,
22494 }
22495 }
22496 #[inline(always)]
22497 fn any_true_mask64x8(self, a: mask64x8<Self>) -> bool {
22498 let bits = u64::from((a).val) & 255u64;
22499 bits != 0
22500 }
22501 #[inline(always)]
22502 fn all_true_mask64x8(self, a: mask64x8<Self>) -> bool {
22503 let bits = u64::from((a).val) & 255u64;
22504 bits == 255u64
22505 }
22506 #[inline(always)]
22507 fn any_false_mask64x8(self, a: mask64x8<Self>) -> bool {
22508 let bits = u64::from((a).val) & 255u64;
22509 bits != 255u64
22510 }
22511 #[inline(always)]
22512 fn all_false_mask64x8(self, a: mask64x8<Self>) -> bool {
22513 let bits = u64::from((a).val) & 255u64;
22514 bits == 0
22515 }
22516 #[inline(always)]
22517 fn split_mask64x8(self, a: mask64x8<Self>) -> (mask64x4<Self>, mask64x4<Self>) {
22518 let bits = u64::from(a.val);
22519 (
22520 mask64x4 {
22521 val: (bits & 15u64) as _,
22522 simd: self,
22523 },
22524 mask64x4 {
22525 val: ((bits >> 4usize) & 15u64) as _,
22526 simd: self,
22527 },
22528 )
22529 }
22530 #[inline(always)]
22531 fn narrow_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask32x16<Self> {
22532 let low = u64::from(a.val) & 255u64;
22533 let high = u64::from(b.val) & 255u64;
22534 mask32x16 {
22535 val: (low | (high << 8usize)) as _,
22536 simd: self,
22537 }
22538 }
22539}
22540impl<S: Simd> SimdFrom<__mmask16, S> for mask8x16<S> {
22541 #[inline(always)]
22542 fn simd_from(simd: S, arch: __mmask16) -> Self {
22543 Self::from_bitmask(simd, u64::from(arch))
22544 }
22545}
22546impl<S: Simd> From<mask8x16<S>> for __mmask16 {
22547 #[inline(always)]
22548 #[allow(
22549 trivial_numeric_casts,
22550 reason = "generated uniformly for all __mmask widths"
22551 )]
22552 fn from(value: mask8x16<S>) -> Self {
22553 value.to_bitmask() as __mmask16
22554 }
22555}
22556impl<S: Simd> SimdFrom<__mmask8, S> for mask16x8<S> {
22557 #[inline(always)]
22558 fn simd_from(simd: S, arch: __mmask8) -> Self {
22559 Self::from_bitmask(simd, u64::from(arch))
22560 }
22561}
22562impl<S: Simd> From<mask16x8<S>> for __mmask8 {
22563 #[inline(always)]
22564 #[allow(
22565 trivial_numeric_casts,
22566 reason = "generated uniformly for all __mmask widths"
22567 )]
22568 fn from(value: mask16x8<S>) -> Self {
22569 value.to_bitmask() as __mmask8
22570 }
22571}
22572impl<S: Simd> SimdFrom<__mmask8, S> for mask32x4<S> {
22573 #[inline(always)]
22574 fn simd_from(simd: S, arch: __mmask8) -> Self {
22575 Self::from_bitmask(simd, u64::from(arch))
22576 }
22577}
22578impl<S: Simd> From<mask32x4<S>> for __mmask8 {
22579 #[inline(always)]
22580 #[allow(
22581 trivial_numeric_casts,
22582 reason = "generated uniformly for all __mmask widths"
22583 )]
22584 fn from(value: mask32x4<S>) -> Self {
22585 value.to_bitmask() as __mmask8
22586 }
22587}
22588impl<S: Simd> SimdFrom<__mmask8, S> for mask64x2<S> {
22589 #[inline(always)]
22590 fn simd_from(simd: S, arch: __mmask8) -> Self {
22591 Self::from_bitmask(simd, u64::from(arch))
22592 }
22593}
22594impl<S: Simd> From<mask64x2<S>> for __mmask8 {
22595 #[inline(always)]
22596 #[allow(
22597 trivial_numeric_casts,
22598 reason = "generated uniformly for all __mmask widths"
22599 )]
22600 fn from(value: mask64x2<S>) -> Self {
22601 value.to_bitmask() as __mmask8
22602 }
22603}
22604impl<S: Simd> SimdFrom<__mmask32, S> for mask8x32<S> {
22605 #[inline(always)]
22606 fn simd_from(simd: S, arch: __mmask32) -> Self {
22607 Self::from_bitmask(simd, u64::from(arch))
22608 }
22609}
22610impl<S: Simd> From<mask8x32<S>> for __mmask32 {
22611 #[inline(always)]
22612 #[allow(
22613 trivial_numeric_casts,
22614 reason = "generated uniformly for all __mmask widths"
22615 )]
22616 fn from(value: mask8x32<S>) -> Self {
22617 value.to_bitmask() as __mmask32
22618 }
22619}
22620impl<S: Simd> SimdFrom<__mmask16, S> for mask16x16<S> {
22621 #[inline(always)]
22622 fn simd_from(simd: S, arch: __mmask16) -> Self {
22623 Self::from_bitmask(simd, u64::from(arch))
22624 }
22625}
22626impl<S: Simd> From<mask16x16<S>> for __mmask16 {
22627 #[inline(always)]
22628 #[allow(
22629 trivial_numeric_casts,
22630 reason = "generated uniformly for all __mmask widths"
22631 )]
22632 fn from(value: mask16x16<S>) -> Self {
22633 value.to_bitmask() as __mmask16
22634 }
22635}
22636impl<S: Simd> SimdFrom<__mmask8, S> for mask32x8<S> {
22637 #[inline(always)]
22638 fn simd_from(simd: S, arch: __mmask8) -> Self {
22639 Self::from_bitmask(simd, u64::from(arch))
22640 }
22641}
22642impl<S: Simd> From<mask32x8<S>> for __mmask8 {
22643 #[inline(always)]
22644 #[allow(
22645 trivial_numeric_casts,
22646 reason = "generated uniformly for all __mmask widths"
22647 )]
22648 fn from(value: mask32x8<S>) -> Self {
22649 value.to_bitmask() as __mmask8
22650 }
22651}
22652impl<S: Simd> SimdFrom<__mmask8, S> for mask64x4<S> {
22653 #[inline(always)]
22654 fn simd_from(simd: S, arch: __mmask8) -> Self {
22655 Self::from_bitmask(simd, u64::from(arch))
22656 }
22657}
22658impl<S: Simd> From<mask64x4<S>> for __mmask8 {
22659 #[inline(always)]
22660 #[allow(
22661 trivial_numeric_casts,
22662 reason = "generated uniformly for all __mmask widths"
22663 )]
22664 fn from(value: mask64x4<S>) -> Self {
22665 value.to_bitmask() as __mmask8
22666 }
22667}
22668impl<S: Simd> SimdFrom<__m512, S> for f32x16<S> {
22669 #[inline(always)]
22670 fn simd_from(simd: S, arch: __m512) -> Self {
22671 Self {
22672 val: crate::transmute::checked_transmute_copy(&arch),
22673 simd,
22674 }
22675 }
22676}
22677impl<S: Simd> From<f32x16<S>> for __m512 {
22678 #[inline(always)]
22679 fn from(value: f32x16<S>) -> Self {
22680 crate::transmute::checked_transmute_copy(&value.val)
22681 }
22682}
22683impl<S: Simd> SimdFrom<__m512i, S> for i8x64<S> {
22684 #[inline(always)]
22685 fn simd_from(simd: S, arch: __m512i) -> Self {
22686 Self {
22687 val: crate::transmute::checked_transmute_copy(&arch),
22688 simd,
22689 }
22690 }
22691}
22692impl<S: Simd> From<i8x64<S>> for __m512i {
22693 #[inline(always)]
22694 fn from(value: i8x64<S>) -> Self {
22695 crate::transmute::checked_transmute_copy(&value.val)
22696 }
22697}
22698impl<S: Simd> SimdFrom<__m512i, S> for u8x64<S> {
22699 #[inline(always)]
22700 fn simd_from(simd: S, arch: __m512i) -> Self {
22701 Self {
22702 val: crate::transmute::checked_transmute_copy(&arch),
22703 simd,
22704 }
22705 }
22706}
22707impl<S: Simd> From<u8x64<S>> for __m512i {
22708 #[inline(always)]
22709 fn from(value: u8x64<S>) -> Self {
22710 crate::transmute::checked_transmute_copy(&value.val)
22711 }
22712}
22713impl<S: Simd> SimdFrom<__mmask64, S> for mask8x64<S> {
22714 #[inline(always)]
22715 fn simd_from(simd: S, arch: __mmask64) -> Self {
22716 Self::from_bitmask(simd, u64::from(arch))
22717 }
22718}
22719impl<S: Simd> From<mask8x64<S>> for __mmask64 {
22720 #[inline(always)]
22721 #[allow(
22722 trivial_numeric_casts,
22723 reason = "generated uniformly for all __mmask widths"
22724 )]
22725 fn from(value: mask8x64<S>) -> Self {
22726 value.to_bitmask() as __mmask64
22727 }
22728}
22729impl<S: Simd> SimdFrom<__m512i, S> for i16x32<S> {
22730 #[inline(always)]
22731 fn simd_from(simd: S, arch: __m512i) -> Self {
22732 Self {
22733 val: crate::transmute::checked_transmute_copy(&arch),
22734 simd,
22735 }
22736 }
22737}
22738impl<S: Simd> From<i16x32<S>> for __m512i {
22739 #[inline(always)]
22740 fn from(value: i16x32<S>) -> Self {
22741 crate::transmute::checked_transmute_copy(&value.val)
22742 }
22743}
22744impl<S: Simd> SimdFrom<__m512i, S> for u16x32<S> {
22745 #[inline(always)]
22746 fn simd_from(simd: S, arch: __m512i) -> Self {
22747 Self {
22748 val: crate::transmute::checked_transmute_copy(&arch),
22749 simd,
22750 }
22751 }
22752}
22753impl<S: Simd> From<u16x32<S>> for __m512i {
22754 #[inline(always)]
22755 fn from(value: u16x32<S>) -> Self {
22756 crate::transmute::checked_transmute_copy(&value.val)
22757 }
22758}
22759impl<S: Simd> SimdFrom<__mmask32, S> for mask16x32<S> {
22760 #[inline(always)]
22761 fn simd_from(simd: S, arch: __mmask32) -> Self {
22762 Self::from_bitmask(simd, u64::from(arch))
22763 }
22764}
22765impl<S: Simd> From<mask16x32<S>> for __mmask32 {
22766 #[inline(always)]
22767 #[allow(
22768 trivial_numeric_casts,
22769 reason = "generated uniformly for all __mmask widths"
22770 )]
22771 fn from(value: mask16x32<S>) -> Self {
22772 value.to_bitmask() as __mmask32
22773 }
22774}
22775impl<S: Simd> SimdFrom<__m512i, S> for i32x16<S> {
22776 #[inline(always)]
22777 fn simd_from(simd: S, arch: __m512i) -> Self {
22778 Self {
22779 val: crate::transmute::checked_transmute_copy(&arch),
22780 simd,
22781 }
22782 }
22783}
22784impl<S: Simd> From<i32x16<S>> for __m512i {
22785 #[inline(always)]
22786 fn from(value: i32x16<S>) -> Self {
22787 crate::transmute::checked_transmute_copy(&value.val)
22788 }
22789}
22790impl<S: Simd> SimdFrom<__m512i, S> for u32x16<S> {
22791 #[inline(always)]
22792 fn simd_from(simd: S, arch: __m512i) -> Self {
22793 Self {
22794 val: crate::transmute::checked_transmute_copy(&arch),
22795 simd,
22796 }
22797 }
22798}
22799impl<S: Simd> From<u32x16<S>> for __m512i {
22800 #[inline(always)]
22801 fn from(value: u32x16<S>) -> Self {
22802 crate::transmute::checked_transmute_copy(&value.val)
22803 }
22804}
22805impl<S: Simd> SimdFrom<__mmask16, S> for mask32x16<S> {
22806 #[inline(always)]
22807 fn simd_from(simd: S, arch: __mmask16) -> Self {
22808 Self::from_bitmask(simd, u64::from(arch))
22809 }
22810}
22811impl<S: Simd> From<mask32x16<S>> for __mmask16 {
22812 #[inline(always)]
22813 #[allow(
22814 trivial_numeric_casts,
22815 reason = "generated uniformly for all __mmask widths"
22816 )]
22817 fn from(value: mask32x16<S>) -> Self {
22818 value.to_bitmask() as __mmask16
22819 }
22820}
22821impl<S: Simd> SimdFrom<__m512d, S> for f64x8<S> {
22822 #[inline(always)]
22823 fn simd_from(simd: S, arch: __m512d) -> Self {
22824 Self {
22825 val: crate::transmute::checked_transmute_copy(&arch),
22826 simd,
22827 }
22828 }
22829}
22830impl<S: Simd> From<f64x8<S>> for __m512d {
22831 #[inline(always)]
22832 fn from(value: f64x8<S>) -> Self {
22833 crate::transmute::checked_transmute_copy(&value.val)
22834 }
22835}
22836impl<S: Simd> SimdFrom<__m512i, S> for i64x8<S> {
22837 #[inline(always)]
22838 fn simd_from(simd: S, arch: __m512i) -> Self {
22839 Self {
22840 val: crate::transmute::checked_transmute_copy(&arch),
22841 simd,
22842 }
22843 }
22844}
22845impl<S: Simd> From<i64x8<S>> for __m512i {
22846 #[inline(always)]
22847 fn from(value: i64x8<S>) -> Self {
22848 crate::transmute::checked_transmute_copy(&value.val)
22849 }
22850}
22851impl<S: Simd> SimdFrom<__m512i, S> for u64x8<S> {
22852 #[inline(always)]
22853 fn simd_from(simd: S, arch: __m512i) -> Self {
22854 Self {
22855 val: crate::transmute::checked_transmute_copy(&arch),
22856 simd,
22857 }
22858 }
22859}
22860impl<S: Simd> From<u64x8<S>> for __m512i {
22861 #[inline(always)]
22862 fn from(value: u64x8<S>) -> Self {
22863 crate::transmute::checked_transmute_copy(&value.val)
22864 }
22865}
22866impl<S: Simd> SimdFrom<__mmask8, S> for mask64x8<S> {
22867 #[inline(always)]
22868 fn simd_from(simd: S, arch: __mmask8) -> Self {
22869 Self::from_bitmask(simd, u64::from(arch))
22870 }
22871}
22872impl<S: Simd> From<mask64x8<S>> for __mmask8 {
22873 #[inline(always)]
22874 #[allow(
22875 trivial_numeric_casts,
22876 reason = "generated uniformly for all __mmask widths"
22877 )]
22878 fn from(value: mask64x8<S>) -> Self {
22879 value.to_bitmask() as __mmask8
22880 }
22881}
22882crate::kernel!(
22883 #[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"]
22884 #[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
22885 #[doc = r" Rust doesn't currently let you do math on const generics."]
22886 #[inline(always)]
22887 fn dyn_alignr_128(token: Avx512, a: __m128i, b: __m128i, shift: usize) -> __m128i {
22888 match shift {
22889 0usize => _mm_alignr_epi8::<0i32>(a, b),
22890 1usize => _mm_alignr_epi8::<1i32>(a, b),
22891 2usize => _mm_alignr_epi8::<2i32>(a, b),
22892 3usize => _mm_alignr_epi8::<3i32>(a, b),
22893 4usize => _mm_alignr_epi8::<4i32>(a, b),
22894 5usize => _mm_alignr_epi8::<5i32>(a, b),
22895 6usize => _mm_alignr_epi8::<6i32>(a, b),
22896 7usize => _mm_alignr_epi8::<7i32>(a, b),
22897 8usize => _mm_alignr_epi8::<8i32>(a, b),
22898 9usize => _mm_alignr_epi8::<9i32>(a, b),
22899 10usize => _mm_alignr_epi8::<10i32>(a, b),
22900 11usize => _mm_alignr_epi8::<11i32>(a, b),
22901 12usize => _mm_alignr_epi8::<12i32>(a, b),
22902 13usize => _mm_alignr_epi8::<13i32>(a, b),
22903 14usize => _mm_alignr_epi8::<14i32>(a, b),
22904 15usize => _mm_alignr_epi8::<15i32>(a, b),
22905 _ => unreachable!(),
22906 }
22907 }
22908);
22909crate::kernel!(
22910 #[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"]
22911 #[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
22912 #[doc = r" Rust doesn't currently let you do math on const generics."]
22913 #[inline(always)]
22914 fn dyn_alignr_256(token: Avx512, a: __m256i, b: __m256i, shift: usize) -> __m256i {
22915 match shift {
22916 0usize => _mm256_alignr_epi8::<0i32>(a, b),
22917 1usize => _mm256_alignr_epi8::<1i32>(a, b),
22918 2usize => _mm256_alignr_epi8::<2i32>(a, b),
22919 3usize => _mm256_alignr_epi8::<3i32>(a, b),
22920 4usize => _mm256_alignr_epi8::<4i32>(a, b),
22921 5usize => _mm256_alignr_epi8::<5i32>(a, b),
22922 6usize => _mm256_alignr_epi8::<6i32>(a, b),
22923 7usize => _mm256_alignr_epi8::<7i32>(a, b),
22924 8usize => _mm256_alignr_epi8::<8i32>(a, b),
22925 9usize => _mm256_alignr_epi8::<9i32>(a, b),
22926 10usize => _mm256_alignr_epi8::<10i32>(a, b),
22927 11usize => _mm256_alignr_epi8::<11i32>(a, b),
22928 12usize => _mm256_alignr_epi8::<12i32>(a, b),
22929 13usize => _mm256_alignr_epi8::<13i32>(a, b),
22930 14usize => _mm256_alignr_epi8::<14i32>(a, b),
22931 15usize => _mm256_alignr_epi8::<15i32>(a, b),
22932 _ => unreachable!(),
22933 }
22934 }
22935);
22936crate::kernel!(
22937 #[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"]
22938 #[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
22939 #[doc = r" Rust doesn't currently let you do math on const generics."]
22940 #[inline(always)]
22941 fn dyn_alignr_512(token: Avx512, a: __m512i, b: __m512i, shift: usize) -> __m512i {
22942 match shift {
22943 0usize => _mm512_alignr_epi8::<0i32>(a, b),
22944 1usize => _mm512_alignr_epi8::<1i32>(a, b),
22945 2usize => _mm512_alignr_epi8::<2i32>(a, b),
22946 3usize => _mm512_alignr_epi8::<3i32>(a, b),
22947 4usize => _mm512_alignr_epi8::<4i32>(a, b),
22948 5usize => _mm512_alignr_epi8::<5i32>(a, b),
22949 6usize => _mm512_alignr_epi8::<6i32>(a, b),
22950 7usize => _mm512_alignr_epi8::<7i32>(a, b),
22951 8usize => _mm512_alignr_epi8::<8i32>(a, b),
22952 9usize => _mm512_alignr_epi8::<9i32>(a, b),
22953 10usize => _mm512_alignr_epi8::<10i32>(a, b),
22954 11usize => _mm512_alignr_epi8::<11i32>(a, b),
22955 12usize => _mm512_alignr_epi8::<12i32>(a, b),
22956 13usize => _mm512_alignr_epi8::<13i32>(a, b),
22957 14usize => _mm512_alignr_epi8::<14i32>(a, b),
22958 15usize => _mm512_alignr_epi8::<15i32>(a, b),
22959 _ => unreachable!(),
22960 }
22961 }
22962);