1use std::arch::x86_64::*;
2use std::mem::{transmute, transmute_copy};
3
4use crate::architectures::interface::*;
5use crate::architectures::macros::*;
6
7#[derive(Copy, Clone)]
8#[repr(transparent)]
9pub struct Avx512Reg(pub __m512i);
10impl SimdArch for Avx512Reg {}
11
12#[derive(Copy, Clone)]
13pub struct Avx512Mask(pub __mmask64);
14impl MaskArch for Avx512Mask {}
15
16impl SimdAddImpl for Avx512Reg {
17 #[inline(always)]
18 unsafe fn f64_add(self, rhs: Self) -> Self {
19 self_from_op!(_mm512_add_pd, self, rhs)
20 }
21 #[inline(always)]
22 unsafe fn f32_add(self, rhs: Self) -> Self {
23 self_from_op!(_mm512_add_ps, self, rhs)
24 }
25 #[inline(always)]
26 unsafe fn i64_add(self, rhs: Self) -> Self {
27 self_from_op!(_mm512_add_epi64, self, rhs)
28 }
29 #[inline(always)]
30 unsafe fn i32_add(self, rhs: Self) -> Self {
31 self_from_op!(_mm512_add_epi32, self, rhs)
32 }
33 #[inline(always)]
34 unsafe fn i16_add(self, rhs: Self) -> Self {
35 self_from_op!(_mm512_add_epi16, self, rhs)
36 }
37 #[inline(always)]
38 unsafe fn i8_add(self, rhs: Self) -> Self {
39 self_from_op!(_mm512_add_epi8, self, rhs)
40 }
41}
42
43impl SimdSubImpl for Avx512Reg {
44 #[inline(always)]
45 unsafe fn f64_sub(self, rhs: Self) -> Self {
46 self_from_op!(_mm512_sub_pd, self, rhs)
47 }
48 #[inline(always)]
49 unsafe fn f32_sub(self, rhs: Self) -> Self {
50 self_from_op!(_mm512_sub_ps, self, rhs)
51 }
52 #[inline(always)]
53 unsafe fn i64_sub(self, rhs: Self) -> Self {
54 self_from_op!(_mm512_sub_epi64, self, rhs)
55 }
56 #[inline(always)]
57 unsafe fn i32_sub(self, rhs: Self) -> Self {
58 self_from_op!(_mm512_sub_epi32, self, rhs)
59 }
60 #[inline(always)]
61 unsafe fn i16_sub(self, rhs: Self) -> Self {
62 self_from_op!(_mm512_sub_epi16, self, rhs)
63 }
64 #[inline(always)]
65 unsafe fn i8_sub(self, rhs: Self) -> Self {
66 self_from_op!(_mm512_sub_epi8, self, rhs)
67 }
68}
69
70impl SimdMulImpl for Avx512Reg {
71 #[inline(always)]
72 unsafe fn f64_mul(self, rhs: Self) -> Self {
73 self_from_op!(_mm512_mul_pd, self, rhs)
74 }
75 #[inline(always)]
76 unsafe fn f32_mul(self, rhs: Self) -> Self {
77 self_from_op!(_mm512_mul_ps, self, rhs)
78 }
79 #[inline(always)]
80 unsafe fn i32_mul(self, rhs: Self) -> Self {
81 self_from_op!(_mm512_mullo_epi32, self, rhs)
82 }
83 #[inline(always)]
84 unsafe fn i16_mul(self, rhs: Self) -> Self {
85 self_from_op!(_mm512_mullo_epi16, self, rhs)
86 }
87}
88
89impl SimdDivImpl for Avx512Reg {
90 #[inline(always)]
91 unsafe fn f64_div(self, rhs: Self) -> Self {
92 self_from_op!(_mm512_div_pd, self, rhs)
93 }
94 #[inline(always)]
95 unsafe fn f32_div(self, rhs: Self) -> Self {
96 self_from_op!(_mm512_div_ps, self, rhs)
97 }
98}
99
100impl SimdBitwiseImpl for Avx512Reg {
101 #[inline(always)]
102 unsafe fn and(self, rhs: Self) -> Self {
103 self_from_op!(_mm512_and_si512, self, rhs)
104 }
105 #[inline(always)]
106 unsafe fn or(self, rhs: Self) -> Self {
107 self_from_op!(_mm512_or_si512, self, rhs)
108 }
109 #[inline(always)]
110 unsafe fn xor(self, rhs: Self) -> Self {
111 self_from_op!(_mm512_xor_si512, self, rhs)
112 }
113 #[inline(always)]
114 unsafe fn not(self) -> Self {
115 unsafe { Self(self.xor(Self::splat_32(!0)).0) }
116 }
117 #[inline(always)]
118 unsafe fn and_not(self, rhs: Self) -> Self {
119 self_from_op!(_mm512_andnot_si512, rhs, self)
120 }
121}
122
123impl SimdShiftImpl for Avx512Reg {
124 #[inline(always)]
125 unsafe fn sllv_64(self, rhs: Self) -> Self {
126 self_from_op!(_mm512_sllv_epi64, self, rhs)
127 }
128 #[inline(always)]
129 unsafe fn srlv_64(self, rhs: Self) -> Self {
130 self_from_op!(_mm512_srlv_epi64, self, rhs)
131 }
132 #[inline(always)]
133 unsafe fn srav_64(self, rhs: Self) -> Self {
134 self_from_op!(_mm512_srav_epi64, self, rhs)
135 }
136 #[inline(always)]
137 unsafe fn sllv_32(self, rhs: Self) -> Self {
138 self_from_op!(_mm512_sllv_epi32, self, rhs)
139 }
140 #[inline(always)]
141 unsafe fn srlv_32(self, rhs: Self) -> Self {
142 self_from_op!(_mm512_srlv_epi32, self, rhs)
143 }
144 #[inline(always)]
145 unsafe fn srav_32(self, rhs: Self) -> Self {
146 self_from_op!(_mm512_srav_epi32, self, rhs)
147 }
148 #[inline(always)]
149 unsafe fn sllv_16(self, rhs: Self) -> Self {
150 self_from_op!(_mm512_sllv_epi16, self, rhs)
151 }
152 #[inline(always)]
153 unsafe fn srlv_16(self, rhs: Self) -> Self {
154 self_from_op!(_mm512_srlv_epi16, self, rhs)
155 }
156 #[inline(always)]
157 unsafe fn srav_16(self, rhs: Self) -> Self {
158 self_from_op!(_mm512_srav_epi16, self, rhs)
159 }
160}
161
162impl SimdLoadImpl for Avx512Reg {
163 type MaskType = Avx512Mask;
164 #[inline(always)]
165 unsafe fn load_aligned<T>(ptr: *const T) -> Self {
166 self_from_op!(_mm512_load_si512, ptr)
167 }
168 #[inline(always)]
169 unsafe fn load_unaligned<T>(ptr: *const T) -> Self {
170 self_from_op!(_mm512_loadu_si512, ptr)
171 }
172 #[inline(always)]
173 unsafe fn masked_load_64<T>(ptr: *const T, mask: Self::MaskType) -> Self {
174 self_from_op!(_mm512_mask_loadu_epi64, Self::zero(), mask, ptr)
175 }
176 #[inline(always)]
177 unsafe fn masked_load_32<T>(ptr: *const T, mask: Self::MaskType) -> Self {
178 self_from_op!(_mm512_mask_loadu_epi32, Self::zero(), mask, ptr)
179 }
180}
181
182impl SimdStoreImpl for Avx512Reg {
183 type MaskType = Avx512Mask;
184 #[inline(always)]
185 unsafe fn store_aligned<T>(self, ptr: *mut T) {
186 execute_intrinsic!(_mm512_store_si512, ptr, self);
187 }
188 #[inline(always)]
189 unsafe fn store_unaligned<T>(self, ptr: *mut T) {
190 execute_intrinsic!(_mm512_storeu_si512, ptr, self);
191 }
192 #[inline(always)]
193 unsafe fn masked_store_64<T>(self, ptr: *mut T, mask: Self::MaskType) {
194 execute_intrinsic!(_mm512_mask_storeu_epi64, ptr, mask, self);
195 }
196 #[inline(always)]
197 unsafe fn masked_store_32<T>(self, ptr: *mut T, mask: Self::MaskType) {
198 execute_intrinsic!(_mm512_mask_storeu_epi32, ptr, mask, self);
199 }
200}
201
202impl SimdZeroImpl for Avx512Reg {
203 #[inline(always)]
204 unsafe fn zero() -> Self {
205 self_from_op!(_mm512_setzero_si512,)
206 }
207}
208
209impl SimdFloatCastsImpl for Avx512Reg {
210 #[inline(always)]
211 unsafe fn float_to_int_trunc(self) -> Self {
212 self_from_op!(_mm512_cvttps_epi32, self)
213 }
214 #[inline(always)]
215 unsafe fn float_to_int_round(self) -> Self {
216 self_from_op!(_mm512_cvtps_epi32, self)
217 }
218}
219
220impl SimdIntCastsImpl for Avx512Reg {
221 #[inline(always)]
222 unsafe fn int_to_float(self) -> Self {
223 self_from_op!(_mm512_cvtepi32_ps, self)
224 }
225}
226
227impl SimdPermuteImpl for Avx512Reg {
228 #[inline(always)]
230 unsafe fn permute_32(self, rhs: Self) -> Self {
231 self_from_op!(_mm512_permutexvar_epi32, rhs, self)
232 }
233 #[inline(always)]
234 unsafe fn permute_8(self, rhs: Self) -> Self {
235 self_from_op!(_mm512_shuffle_epi8, self, rhs)
236 }
237}
238
239impl SimdVariableBlendImpl for Avx512Mask {
240 type VecType = Avx512Reg;
241 #[inline(always)]
242 unsafe fn vblend_64(self, true_values: Self::VecType, false_values: Self::VecType) -> Self::VecType {
243 unsafe {
244 Avx512Reg(transmute(execute_intrinsic!(
245 _mm512_mask_blend_pd,
246 self,
247 false_values,
248 true_values
249 )))
250 }
251 }
252 #[inline(always)]
253 unsafe fn vblend_32(self, true_values: Self::VecType, false_values: Self::VecType) -> Self::VecType {
254 unsafe {
255 Avx512Reg(transmute(execute_intrinsic!(
256 _mm512_mask_blend_ps,
257 self,
258 false_values,
259 true_values
260 )))
261 }
262 }
263 #[inline(always)]
264 unsafe fn vblend_8(self, true_values: Self::VecType, false_values: Self::VecType) -> Self::VecType {
265 unsafe {
266 Avx512Reg(transmute(execute_intrinsic!(
267 _mm512_mask_blend_epi8,
268 self,
269 false_values,
270 true_values
271 )))
272 }
273 }
274}
275
276impl SimdImmediateBlendImpl for Avx512Reg {
277 #[inline(always)]
278 unsafe fn blend_64<const N: i32>(self, false_values: Self) -> Self {
279 let mask = Avx512Mask(N as u64);
280 unsafe { mask.vblend_64(self, false_values) }
281 }
282 #[inline(always)]
283 unsafe fn blend_32<const N: i32>(self, false_values: Self) -> Self {
284 let mask = Avx512Mask(N as u64);
285 unsafe { mask.vblend_32(self, false_values) }
286 }
287}
288
289impl SimdMulAddImpl for Avx512Reg {
290 #[inline(always)]
291 unsafe fn mul_add_f64(self, mult: Self, add: Self) -> Self {
292 self_from_op!(_mm512_fmadd_pd, self, mult, add)
293 }
294 #[inline(always)]
295 unsafe fn mul_sub_f64(self, mult: Self, sub: Self) -> Self {
296 self_from_op!(_mm512_fmsub_pd, self, mult, sub)
297 }
298 #[inline(always)]
299 unsafe fn negated_mul_add_f64(self, mult: Self, add: Self) -> Self {
300 self_from_op!(_mm512_fnmadd_pd, self, mult, add)
301 }
302 #[inline(always)]
303 unsafe fn negated_mul_sub_f64(self, mult: Self, sub: Self) -> Self {
304 self_from_op!(_mm512_fnmsub_pd, self, mult, sub)
305 }
306 #[inline(always)]
307 unsafe fn mul_add_f32(self, mult: Self, add: Self) -> Self {
308 self_from_op!(_mm512_fmadd_ps, self, mult, add)
309 }
310 #[inline(always)]
311 unsafe fn mul_sub_f32(self, mult: Self, sub: Self) -> Self {
312 self_from_op!(_mm512_fmsub_ps, self, mult, sub)
313 }
314 #[inline(always)]
315 unsafe fn negated_mul_add_f32(self, mult: Self, add: Self) -> Self {
316 self_from_op!(_mm512_fnmadd_ps, self, mult, add)
317 }
318 #[inline(always)]
319 unsafe fn negated_mul_sub_f32(self, mult: Self, sub: Self) -> Self {
320 self_from_op!(_mm512_fnmsub_ps, self, mult, sub)
321 }
322}
323
324impl SimdRoundImpl for Avx512Reg {
325 #[inline(always)]
326 unsafe fn round_f64(self) -> Self {
327 self_from_const_op!(
328 _mm512_roundscale_pd,
329 _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC,
330 self
331 )
332 }
333 #[inline(always)]
334 unsafe fn round_f32(self) -> Self {
335 self_from_const_op!(
336 _mm512_roundscale_ps,
337 _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC,
338 self
339 )
340 }
341 #[inline(always)]
342 unsafe fn floor_f64(self) -> Self {
343 self_from_const_op!(
344 _mm512_roundscale_pd,
345 _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC,
346 self
347 )
348 }
349 #[inline(always)]
350 unsafe fn floor_f32(self) -> Self {
351 self_from_const_op!(
352 _mm512_roundscale_ps,
353 _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC,
354 self
355 )
356 }
357 #[inline(always)]
358 unsafe fn ceil_f64(self) -> Self {
359 self_from_const_op!(
360 _mm512_roundscale_pd,
361 _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC,
362 self
363 )
364 }
365 #[inline(always)]
366 unsafe fn ceil_f32(self) -> Self {
367 self_from_const_op!(
368 _mm512_roundscale_ps,
369 _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC,
370 self
371 )
372 }
373}
374
375impl SimdPartialOrdImpl for Avx512Reg {
376 type MaskType = Avx512Mask;
377 #[inline(always)]
378 unsafe fn cmp_f64_eq(self, rhs: Self) -> Self::MaskType {
379 Avx512Mask(execute_const_intrinsic!(_mm512_cmp_pd_mask, _CMP_EQ_OQ, self, rhs) as u64)
380 }
381 #[inline(always)]
382 unsafe fn cmp_f64_lt(self, rhs: Self) -> Self::MaskType {
383 Avx512Mask(execute_const_intrinsic!(_mm512_cmp_pd_mask, _CMP_LT_OQ, self, rhs) as u64)
384 }
385 #[inline(always)]
386 unsafe fn cmp_f64_le(self, rhs: Self) -> Self::MaskType {
387 Avx512Mask(execute_const_intrinsic!(_mm512_cmp_pd_mask, _CMP_LE_OQ, self, rhs) as u64)
388 }
389 #[inline(always)]
390 unsafe fn cmp_f64_gt(self, rhs: Self) -> Self::MaskType {
391 Avx512Mask(execute_const_intrinsic!(_mm512_cmp_pd_mask, _CMP_GT_OQ, self, rhs) as u64)
392 }
393 #[inline(always)]
394 unsafe fn cmp_f64_ge(self, rhs: Self) -> Self::MaskType {
395 Avx512Mask(execute_const_intrinsic!(_mm512_cmp_pd_mask, _CMP_GE_OQ, self, rhs) as u64)
396 }
397 #[inline(always)]
398 unsafe fn cmp_f64_neq(self, rhs: Self) -> Self::MaskType {
399 Avx512Mask(execute_const_intrinsic!(_mm512_cmp_pd_mask, _CMP_NEQ_OQ, self, rhs) as u64)
400 }
401 #[inline(always)]
402 unsafe fn cmp_f32_eq(self, rhs: Self) -> Self::MaskType {
403 Avx512Mask(execute_const_intrinsic!(_mm512_cmp_ps_mask, _CMP_EQ_OQ, self, rhs) as u64)
404 }
405 #[inline(always)]
406 unsafe fn cmp_f32_lt(self, rhs: Self) -> Self::MaskType {
407 Avx512Mask(execute_const_intrinsic!(_mm512_cmp_ps_mask, _CMP_LT_OQ, self, rhs) as u64)
408 }
409 #[inline(always)]
410 unsafe fn cmp_f32_le(self, rhs: Self) -> Self::MaskType {
411 Avx512Mask(execute_const_intrinsic!(_mm512_cmp_ps_mask, _CMP_LE_OQ, self, rhs) as u64)
412 }
413 #[inline(always)]
414 unsafe fn cmp_f32_gt(self, rhs: Self) -> Self::MaskType {
415 Avx512Mask(execute_const_intrinsic!(_mm512_cmp_ps_mask, _CMP_GT_OQ, self, rhs) as u64)
416 }
417 #[inline(always)]
418 unsafe fn cmp_f32_ge(self, rhs: Self) -> Self::MaskType {
419 Avx512Mask(execute_const_intrinsic!(_mm512_cmp_ps_mask, _CMP_GE_OQ, self, rhs) as u64)
420 }
421 #[inline(always)]
422 unsafe fn cmp_f32_neq(self, rhs: Self) -> Self::MaskType {
423 Avx512Mask(execute_const_intrinsic!(_mm512_cmp_ps_mask, _CMP_NEQ_OQ, self, rhs) as u64)
424 }
425 #[inline(always)]
426 unsafe fn cmp_i64_eq(self, rhs: Self) -> Self::MaskType {
427 Avx512Mask(execute_intrinsic!(_mm512_cmpeq_epi64_mask, self, rhs) as u64)
428 }
429 #[inline(always)]
430 unsafe fn cmp_i64_gt(self, rhs: Self) -> Self::MaskType {
431 Avx512Mask(execute_intrinsic!(_mm512_cmpgt_epi64_mask, self, rhs) as u64)
432 }
433 #[inline(always)]
434 unsafe fn cmp_i32_eq(self, rhs: Self) -> Self::MaskType {
435 Avx512Mask(execute_intrinsic!(_mm512_cmpeq_epi32_mask, self, rhs) as u64)
436 }
437 #[inline(always)]
438 unsafe fn cmp_i32_gt(self, rhs: Self) -> Self::MaskType {
439 Avx512Mask(execute_intrinsic!(_mm512_cmpgt_epi32_mask, self, rhs) as u64)
440 }
441 #[inline(always)]
442 unsafe fn cmp_i16_eq(self, rhs: Self) -> Self::MaskType {
443 Avx512Mask(execute_intrinsic!(_mm512_cmpeq_epi16_mask, self, rhs) as u64)
444 }
445 #[inline(always)]
446 unsafe fn cmp_i16_gt(self, rhs: Self) -> Self::MaskType {
447 Avx512Mask(execute_intrinsic!(_mm512_cmpgt_epi16_mask, self, rhs) as u64)
448 }
449 #[inline(always)]
450 unsafe fn cmp_i8_eq(self, rhs: Self) -> Self::MaskType {
451 Avx512Mask(execute_intrinsic!(_mm512_cmpeq_epi8_mask, self, rhs) as u64)
452 }
453 #[inline(always)]
454 unsafe fn cmp_i8_gt(self, rhs: Self) -> Self::MaskType {
455 Avx512Mask(execute_intrinsic!(_mm512_cmpgt_epi8_mask, self, rhs) as u64)
456 }
457
458 #[inline(always)]
459 unsafe fn max_f64(self, rhs: Self) -> Self {
460 self_from_op!(_mm512_max_pd, self, rhs)
461 }
462 #[inline(always)]
463 unsafe fn min_f64(self, rhs: Self) -> Self {
464 self_from_op!(_mm512_min_pd, self, rhs)
465 }
466 #[inline(always)]
467 unsafe fn max_f32(self, rhs: Self) -> Self {
468 self_from_op!(_mm512_max_ps, self, rhs)
469 }
470 #[inline(always)]
471 unsafe fn min_f32(self, rhs: Self) -> Self {
472 self_from_op!(_mm512_min_ps, self, rhs)
473 }
474 #[inline(always)]
475 unsafe fn max_i32(self, rhs: Self) -> Self {
476 self_from_op!(_mm512_max_epi32, self, rhs)
477 }
478 #[inline(always)]
479 unsafe fn min_i32(self, rhs: Self) -> Self {
480 self_from_op!(_mm512_min_epi32, self, rhs)
481 }
482 #[inline(always)]
483 unsafe fn max_i16(self, rhs: Self) -> Self {
484 self_from_op!(_mm512_max_epi16, self, rhs)
485 }
486 #[inline(always)]
487 unsafe fn min_i16(self, rhs: Self) -> Self {
488 self_from_op!(_mm512_min_epi16, self, rhs)
489 }
490 #[inline(always)]
491 unsafe fn max_i8(self, rhs: Self) -> Self {
492 self_from_op!(_mm512_max_epi8, self, rhs)
493 }
494 #[inline(always)]
495 unsafe fn min_i8(self, rhs: Self) -> Self {
496 self_from_op!(_mm512_min_epi8, self, rhs)
497 }
498 #[inline(always)]
499 unsafe fn max_u32(self, rhs: Self) -> Self {
500 self_from_op!(_mm512_max_epu32, self, rhs)
501 }
502 #[inline(always)]
503 unsafe fn min_u32(self, rhs: Self) -> Self {
504 self_from_op!(_mm512_min_epu32, self, rhs)
505 }
506 #[inline(always)]
507 unsafe fn max_u16(self, rhs: Self) -> Self {
508 self_from_op!(_mm512_max_epu16, self, rhs)
509 }
510 #[inline(always)]
511 unsafe fn min_u16(self, rhs: Self) -> Self {
512 self_from_op!(_mm512_min_epu16, self, rhs)
513 }
514 #[inline(always)]
515 unsafe fn max_u8(self, rhs: Self) -> Self {
516 self_from_op!(_mm512_max_epu8, self, rhs)
517 }
518 #[inline(always)]
519 unsafe fn min_u8(self, rhs: Self) -> Self {
520 self_from_op!(_mm512_min_epu8, self, rhs)
521 }
522}
523
524impl SimdSplatImpl for Avx512Reg {
526 #[inline(always)]
527 unsafe fn splat_64<T>(val: T) -> Self {
528 self_from_op!(_mm512_set1_epi64, val)
529 }
530 #[inline(always)]
531 unsafe fn splat_32<T>(val: T) -> Self {
532 self_from_op!(_mm512_set1_epi32, val)
533 }
534 #[inline(always)]
535 unsafe fn splat_16<T>(val: T) -> Self {
536 self_from_op!(_mm512_set1_epi16, val)
537 }
538 #[inline(always)]
539 unsafe fn splat_8<T>(val: T) -> Self {
540 self_from_op!(_mm512_set1_epi8, val)
541 }
542}
543
544impl SimdBitwiseImpl for Avx512Mask {
545 #[inline(always)]
546 unsafe fn and(self, rhs: Self) -> Self {
547 self_from_op!(_kand_mask64, self, rhs)
548 }
549 #[inline(always)]
550 unsafe fn or(self, rhs: Self) -> Self {
551 self_from_op!(_kor_mask64, self, rhs)
552 }
553 #[inline(always)]
554 unsafe fn xor(self, rhs: Self) -> Self {
555 self_from_op!(_kxor_mask64, self, rhs)
556 }
557 #[inline(always)]
558 unsafe fn not(self) -> Self {
559 self_from_op!(_knot_mask64, self)
560 }
561 #[inline(always)]
562 unsafe fn and_not(self, rhs: Self) -> Self {
563 self_from_op!(_kandn_mask64, rhs, self)
564 }
565}
566
567impl SimdGatherImpl for Avx512Reg {
568 #[inline(always)]
569 unsafe fn gather_32_from_32<T, const B: i32>(self, ptr: *const T) -> Self {
570 self_from_const_op!(_mm512_i32gather_epi32, B, self, ptr)
571 }
572 #[inline(always)]
575 unsafe fn gather_64_from_64<T, const B: i32>(self, ptr: *const T) -> Self {
576 self_from_const_op!(_mm512_i64gather_epi64, B, self, ptr)
577 }
578}
579
580impl SimdSqrtImpl for Avx512Reg {
581 #[inline(always)]
582 unsafe fn sqrt_f64(self) -> Self {
583 self_from_op!(_mm512_sqrt_pd, self)
584 }
585 #[inline(always)]
586 unsafe fn sqrt_f32(self) -> Self {
587 self_from_op!(_mm512_sqrt_ps, self)
588 }
589 #[inline(always)]
590 unsafe fn rsqrt_f32(self) -> Self {
591 self_from_op!(_mm512_rsqrt14_ps, self)
592 }
593}
594
595impl SimdAllBitsImpl for Avx512Mask {
596 #[inline(always)]
597 unsafe fn all_zero(self) -> bool {
598 self.0 == 0
599 }
600}
601
602impl SimdNegateImpl for Avx512Reg {
603 #[inline(always)]
604 unsafe fn negate_f64(self) -> Self {
605 unsafe { Self::splat_64(-0.0f64).xor(self) }
606 }
607 #[inline(always)]
608 unsafe fn negate_f32(self) -> Self {
609 unsafe { Self::splat_32(-0.0f64).xor(self) }
610 }
611}
612
613impl SimdBlockShiftImpl for Avx512Reg {
614 #[inline(always)]
615 unsafe fn block_left_byte_shift<const N: i32>(self) -> Self {
616 self_from_const_op!(_mm512_bslli_epi128, N, self)
617 }
618 #[inline(always)]
619 unsafe fn block_right_byte_shift<const N: i32>(self) -> Self {
620 self_from_const_op!(_mm512_bsrli_epi128, N, self)
621 }
622}
623
624impl SimdMaskBitConversion for Avx512Mask {
625 #[inline(always)]
626 unsafe fn to_bits_64(self) -> u64 {
627 self.0
628 }
629 #[inline(always)]
630 unsafe fn to_bits_32(self) -> u64 {
631 self.0
632 }
633 #[inline(always)]
634 unsafe fn to_bits_8(self) -> u64 {
635 self.0
636 }
637 #[inline(always)]
638 unsafe fn from_bits_64(bitmask: u64) -> Self {
639 unsafe { transmute(bitmask) }
640 }
641 #[inline(always)]
642 unsafe fn from_bits_32(bitmask: u64) -> Self {
643 unsafe { transmute(bitmask) }
644 }
645 #[inline(always)]
646 unsafe fn from_bits_16(bitmask: u64) -> Self {
647 unsafe { transmute(bitmask) }
648 }
649 #[inline(always)]
650 unsafe fn from_bits_8(bitmask: u64) -> Self {
651 unsafe { transmute(bitmask) }
652 }
653}
654
655impl SimdLaneShiftImpl for Avx512Reg {
656 #[inline(always)]
657 unsafe fn right_lane_shift_32(self, n: u32) -> Self {
658 match n {
659 0 => self,
660 1 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 15) }),
661 2 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 14) }),
662 3 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 13) }),
663 4 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 12) }),
664 5 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 11) }),
665 6 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 10) }),
666 7 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 9) }),
667 8 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 8) }),
668 9 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 7) }),
669 10 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 6) }),
670 11 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 5) }),
671 12 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 4) }),
672 13 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 3) }),
673 14 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 2) }),
674 15 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 1) }),
675 _ => unsafe { Self::zero() },
676 }
677 }
678
679 #[inline(always)]
680 unsafe fn left_lane_shift_32(self, n: u32) -> Self {
681 match n {
682 0 => self,
683 1 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 1) }),
684 2 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 2) }),
685 3 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 3) }),
686 4 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 4) }),
687 5 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 5) }),
688 6 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 6) }),
689 7 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 7) }),
690 8 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 8) }),
691 9 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 9) }),
692 10 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 10) }),
693 11 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 11) }),
694 12 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 12) }),
695 13 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 13) }),
696 14 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 14) }),
697 15 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 15) }),
698 _ => unsafe { Self::zero() },
699 }
700 }
701}
702