Skip to main content

simply_simd/architectures/intrinsics/
avx512.rs

1use std::arch::x86_64::*;
2use std::mem::{transmute, transmute_copy};
3
4use crate::architectures::interface::*;
5use crate::architectures::macros::*;
6
7#[derive(Copy, Clone)]
8#[repr(transparent)]
9pub struct Avx512Reg(pub __m512i);
10impl SimdArch for Avx512Reg {}
11
12#[derive(Copy, Clone)]
13pub struct Avx512Mask(pub __mmask64);
14impl MaskArch for Avx512Mask {}
15
16impl SimdAddImpl for Avx512Reg {
17    #[inline(always)]
18    unsafe fn f64_add(self, rhs: Self) -> Self {
19        self_from_op!(_mm512_add_pd, self, rhs)
20    }
21    #[inline(always)]
22    unsafe fn f32_add(self, rhs: Self) -> Self {
23        self_from_op!(_mm512_add_ps, self, rhs)
24    }
25    #[inline(always)]
26    unsafe fn i64_add(self, rhs: Self) -> Self {
27        self_from_op!(_mm512_add_epi64, self, rhs)
28    }
29    #[inline(always)]
30    unsafe fn i32_add(self, rhs: Self) -> Self {
31        self_from_op!(_mm512_add_epi32, self, rhs)
32    }
33    #[inline(always)]
34    unsafe fn i16_add(self, rhs: Self) -> Self {
35        self_from_op!(_mm512_add_epi16, self, rhs)
36    }
37    #[inline(always)]
38    unsafe fn i8_add(self, rhs: Self) -> Self {
39        self_from_op!(_mm512_add_epi8, self, rhs)
40    }
41}
42
43impl SimdSubImpl for Avx512Reg {
44    #[inline(always)]
45    unsafe fn f64_sub(self, rhs: Self) -> Self {
46        self_from_op!(_mm512_sub_pd, self, rhs)
47    }
48    #[inline(always)]
49    unsafe fn f32_sub(self, rhs: Self) -> Self {
50        self_from_op!(_mm512_sub_ps, self, rhs)
51    }
52    #[inline(always)]
53    unsafe fn i64_sub(self, rhs: Self) -> Self {
54        self_from_op!(_mm512_sub_epi64, self, rhs)
55    }
56    #[inline(always)]
57    unsafe fn i32_sub(self, rhs: Self) -> Self {
58        self_from_op!(_mm512_sub_epi32, self, rhs)
59    }
60    #[inline(always)]
61    unsafe fn i16_sub(self, rhs: Self) -> Self {
62        self_from_op!(_mm512_sub_epi16, self, rhs)
63    }
64    #[inline(always)]
65    unsafe fn i8_sub(self, rhs: Self) -> Self {
66        self_from_op!(_mm512_sub_epi8, self, rhs)
67    }
68}
69
70impl SimdMulImpl for Avx512Reg {
71    #[inline(always)]
72    unsafe fn f64_mul(self, rhs: Self) -> Self {
73        self_from_op!(_mm512_mul_pd, self, rhs)
74    }
75    #[inline(always)]
76    unsafe fn f32_mul(self, rhs: Self) -> Self {
77        self_from_op!(_mm512_mul_ps, self, rhs)
78    }
79    #[inline(always)]
80    unsafe fn i32_mul(self, rhs: Self) -> Self {
81        self_from_op!(_mm512_mullo_epi32, self, rhs)
82    }
83    #[inline(always)]
84    unsafe fn i16_mul(self, rhs: Self) -> Self {
85        self_from_op!(_mm512_mullo_epi16, self, rhs)
86    }
87}
88
89impl SimdDivImpl for Avx512Reg {
90    #[inline(always)]
91    unsafe fn f64_div(self, rhs: Self) -> Self {
92        self_from_op!(_mm512_div_pd, self, rhs)
93    }
94    #[inline(always)]
95    unsafe fn f32_div(self, rhs: Self) -> Self {
96        self_from_op!(_mm512_div_ps, self, rhs)
97    }
98}
99
100impl SimdBitwiseImpl for Avx512Reg {
101    #[inline(always)]
102    unsafe fn and(self, rhs: Self) -> Self {
103        self_from_op!(_mm512_and_si512, self, rhs)
104    }
105    #[inline(always)]
106    unsafe fn or(self, rhs: Self) -> Self {
107        self_from_op!(_mm512_or_si512, self, rhs)
108    }
109    #[inline(always)]
110    unsafe fn xor(self, rhs: Self) -> Self {
111        self_from_op!(_mm512_xor_si512, self, rhs)
112    }
113    #[inline(always)]
114    unsafe fn not(self) -> Self {
115        unsafe { Self(self.xor(Self::splat_32(!0)).0) }
116    }
117    #[inline(always)]
118    unsafe fn and_not(self, rhs: Self) -> Self {
119        self_from_op!(_mm512_andnot_si512, rhs, self)
120    }
121}
122
123impl SimdShiftImpl for Avx512Reg {
124    #[inline(always)]
125    unsafe fn sllv_64(self, rhs: Self) -> Self {
126        self_from_op!(_mm512_sllv_epi64, self, rhs)
127    }
128    #[inline(always)]
129    unsafe fn srlv_64(self, rhs: Self) -> Self {
130        self_from_op!(_mm512_srlv_epi64, self, rhs)
131    }
132    #[inline(always)]
133    unsafe fn srav_64(self, rhs: Self) -> Self {
134        self_from_op!(_mm512_srav_epi64, self, rhs)
135    }
136    #[inline(always)]
137    unsafe fn sllv_32(self, rhs: Self) -> Self {
138        self_from_op!(_mm512_sllv_epi32, self, rhs)
139    }
140    #[inline(always)]
141    unsafe fn srlv_32(self, rhs: Self) -> Self {
142        self_from_op!(_mm512_srlv_epi32, self, rhs)
143    }
144    #[inline(always)]
145    unsafe fn srav_32(self, rhs: Self) -> Self {
146        self_from_op!(_mm512_srav_epi32, self, rhs)
147    }
148    #[inline(always)]
149    unsafe fn sllv_16(self, rhs: Self) -> Self {
150        self_from_op!(_mm512_sllv_epi16, self, rhs)
151    }
152    #[inline(always)]
153    unsafe fn srlv_16(self, rhs: Self) -> Self {
154        self_from_op!(_mm512_srlv_epi16, self, rhs)
155    }
156    #[inline(always)]
157    unsafe fn srav_16(self, rhs: Self) -> Self {
158        self_from_op!(_mm512_srav_epi16, self, rhs)
159    }
160}
161
162impl SimdLoadImpl for Avx512Reg {
163    type MaskType = Avx512Mask;
164    #[inline(always)]
165    unsafe fn load_aligned<T>(ptr: *const T) -> Self {
166        self_from_op!(_mm512_load_si512, ptr)
167    }
168    #[inline(always)]
169    unsafe fn load_unaligned<T>(ptr: *const T) -> Self {
170        self_from_op!(_mm512_loadu_si512, ptr)
171    }
172    #[inline(always)]
173    unsafe fn masked_load_64<T>(ptr: *const T, mask: Self::MaskType) -> Self {
174        self_from_op!(_mm512_mask_loadu_epi64, Self::zero(), mask, ptr)
175    }
176    #[inline(always)]
177    unsafe fn masked_load_32<T>(ptr: *const T, mask: Self::MaskType) -> Self {
178        self_from_op!(_mm512_mask_loadu_epi32, Self::zero(), mask, ptr)
179    }
180}
181
182impl SimdStoreImpl for Avx512Reg {
183    type MaskType = Avx512Mask;
184    #[inline(always)]
185    unsafe fn store_aligned<T>(self, ptr: *mut T) {
186        execute_intrinsic!(_mm512_store_si512, ptr, self);
187    }
188    #[inline(always)]
189    unsafe fn store_unaligned<T>(self, ptr: *mut T) {
190        execute_intrinsic!(_mm512_storeu_si512, ptr, self);
191    }
192    #[inline(always)]
193    unsafe fn masked_store_64<T>(self, ptr: *mut T, mask: Self::MaskType) {
194        execute_intrinsic!(_mm512_mask_storeu_epi64, ptr, mask, self);
195    }
196    #[inline(always)]
197    unsafe fn masked_store_32<T>(self, ptr: *mut T, mask: Self::MaskType) {
198        execute_intrinsic!(_mm512_mask_storeu_epi32, ptr, mask, self);
199    }
200}
201
202impl SimdZeroImpl for Avx512Reg {
203    #[inline(always)]
204    unsafe fn zero() -> Self {
205        self_from_op!(_mm512_setzero_si512,)
206    }
207}
208
209impl SimdFloatCastsImpl for Avx512Reg {
210    #[inline(always)]
211    unsafe fn float_to_int_trunc(self) -> Self {
212        self_from_op!(_mm512_cvttps_epi32, self)
213    }
214    #[inline(always)]
215    unsafe fn float_to_int_round(self) -> Self {
216        self_from_op!(_mm512_cvtps_epi32, self)
217    }
218}
219
220impl SimdIntCastsImpl for Avx512Reg {
221    #[inline(always)]
222    unsafe fn int_to_float(self) -> Self {
223        self_from_op!(_mm512_cvtepi32_ps, self)
224    }
225}
226
227impl SimdPermuteImpl for Avx512Reg {
228    // type BlockVec = Sse;
229    #[inline(always)]
230    unsafe fn permute_32(self, rhs: Self) -> Self {
231        self_from_op!(_mm512_permutexvar_epi32, rhs, self)
232    }
233    #[inline(always)]
234    unsafe fn permute_8(self, rhs: Self) -> Self {
235        self_from_op!(_mm512_shuffle_epi8, self, rhs)
236    }
237}
238
239impl SimdVariableBlendImpl for Avx512Mask {
240    type VecType = Avx512Reg;
241    #[inline(always)]
242    unsafe fn vblend_64(self, true_values: Self::VecType, false_values: Self::VecType) -> Self::VecType {
243        unsafe {
244            Avx512Reg(transmute(execute_intrinsic!(
245                _mm512_mask_blend_pd,
246                self,
247                false_values,
248                true_values
249            )))
250        }
251    }
252    #[inline(always)]
253    unsafe fn vblend_32(self, true_values: Self::VecType, false_values: Self::VecType) -> Self::VecType {
254        unsafe {
255            Avx512Reg(transmute(execute_intrinsic!(
256                _mm512_mask_blend_ps,
257                self,
258                false_values,
259                true_values
260            )))
261        }
262    }
263    #[inline(always)]
264    unsafe fn vblend_8(self, true_values: Self::VecType, false_values: Self::VecType) -> Self::VecType {
265        unsafe {
266            Avx512Reg(transmute(execute_intrinsic!(
267                _mm512_mask_blend_epi8,
268                self,
269                false_values,
270                true_values
271            )))
272        }
273    }
274}
275
276impl SimdImmediateBlendImpl for Avx512Reg {
277    #[inline(always)]
278    unsafe fn blend_64<const N: i32>(self, false_values: Self) -> Self {
279        let mask = Avx512Mask(N as u64);
280        unsafe { mask.vblend_64(self, false_values) }
281    }
282    #[inline(always)]
283    unsafe fn blend_32<const N: i32>(self, false_values: Self) -> Self {
284        let mask = Avx512Mask(N as u64);
285        unsafe { mask.vblend_32(self, false_values) }
286    }
287}
288
289impl SimdMulAddImpl for Avx512Reg {
290    #[inline(always)]
291    unsafe fn mul_add_f64(self, mult: Self, add: Self) -> Self {
292        self_from_op!(_mm512_fmadd_pd, self, mult, add)
293    }
294    #[inline(always)]
295    unsafe fn mul_sub_f64(self, mult: Self, sub: Self) -> Self {
296        self_from_op!(_mm512_fmsub_pd, self, mult, sub)
297    }
298    #[inline(always)]
299    unsafe fn negated_mul_add_f64(self, mult: Self, add: Self) -> Self {
300        self_from_op!(_mm512_fnmadd_pd, self, mult, add)
301    }
302    #[inline(always)]
303    unsafe fn negated_mul_sub_f64(self, mult: Self, sub: Self) -> Self {
304        self_from_op!(_mm512_fnmsub_pd, self, mult, sub)
305    }
306    #[inline(always)]
307    unsafe fn mul_add_f32(self, mult: Self, add: Self) -> Self {
308        self_from_op!(_mm512_fmadd_ps, self, mult, add)
309    }
310    #[inline(always)]
311    unsafe fn mul_sub_f32(self, mult: Self, sub: Self) -> Self {
312        self_from_op!(_mm512_fmsub_ps, self, mult, sub)
313    }
314    #[inline(always)]
315    unsafe fn negated_mul_add_f32(self, mult: Self, add: Self) -> Self {
316        self_from_op!(_mm512_fnmadd_ps, self, mult, add)
317    }
318    #[inline(always)]
319    unsafe fn negated_mul_sub_f32(self, mult: Self, sub: Self) -> Self {
320        self_from_op!(_mm512_fnmsub_ps, self, mult, sub)
321    }
322}
323
324impl SimdRoundImpl for Avx512Reg {
325    #[inline(always)]
326    unsafe fn round_f64(self) -> Self {
327        self_from_const_op!(
328            _mm512_roundscale_pd,
329            _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC,
330            self
331        )
332    }
333    #[inline(always)]
334    unsafe fn round_f32(self) -> Self {
335        self_from_const_op!(
336            _mm512_roundscale_ps,
337            _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC,
338            self
339        )
340    }
341    #[inline(always)]
342    unsafe fn floor_f64(self) -> Self {
343        self_from_const_op!(
344            _mm512_roundscale_pd,
345            _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC,
346            self
347        )
348    }
349    #[inline(always)]
350    unsafe fn floor_f32(self) -> Self {
351        self_from_const_op!(
352            _mm512_roundscale_ps,
353            _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC,
354            self
355        )
356    }
357    #[inline(always)]
358    unsafe fn ceil_f64(self) -> Self {
359        self_from_const_op!(
360            _mm512_roundscale_pd,
361            _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC,
362            self
363        )
364    }
365    #[inline(always)]
366    unsafe fn ceil_f32(self) -> Self {
367        self_from_const_op!(
368            _mm512_roundscale_ps,
369            _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC,
370            self
371        )
372    }
373}
374
375impl SimdPartialOrdImpl for Avx512Reg {
376    type MaskType = Avx512Mask;
377    #[inline(always)]
378    unsafe fn cmp_f64_eq(self, rhs: Self) -> Self::MaskType {
379        Avx512Mask(execute_const_intrinsic!(_mm512_cmp_pd_mask, _CMP_EQ_OQ, self, rhs) as u64)
380    }
381    #[inline(always)]
382    unsafe fn cmp_f64_lt(self, rhs: Self) -> Self::MaskType {
383        Avx512Mask(execute_const_intrinsic!(_mm512_cmp_pd_mask, _CMP_LT_OQ, self, rhs) as u64)
384    }
385    #[inline(always)]
386    unsafe fn cmp_f64_le(self, rhs: Self) -> Self::MaskType {
387        Avx512Mask(execute_const_intrinsic!(_mm512_cmp_pd_mask, _CMP_LE_OQ, self, rhs) as u64)
388    }
389    #[inline(always)]
390    unsafe fn cmp_f64_gt(self, rhs: Self) -> Self::MaskType {
391        Avx512Mask(execute_const_intrinsic!(_mm512_cmp_pd_mask, _CMP_GT_OQ, self, rhs) as u64)
392    }
393    #[inline(always)]
394    unsafe fn cmp_f64_ge(self, rhs: Self) -> Self::MaskType {
395        Avx512Mask(execute_const_intrinsic!(_mm512_cmp_pd_mask, _CMP_GE_OQ, self, rhs) as u64)
396    }
397    #[inline(always)]
398    unsafe fn cmp_f64_neq(self, rhs: Self) -> Self::MaskType {
399        Avx512Mask(execute_const_intrinsic!(_mm512_cmp_pd_mask, _CMP_NEQ_OQ, self, rhs) as u64)
400    }
401    #[inline(always)]
402    unsafe fn cmp_f32_eq(self, rhs: Self) -> Self::MaskType {
403        Avx512Mask(execute_const_intrinsic!(_mm512_cmp_ps_mask, _CMP_EQ_OQ, self, rhs) as u64)
404    }
405    #[inline(always)]
406    unsafe fn cmp_f32_lt(self, rhs: Self) -> Self::MaskType {
407        Avx512Mask(execute_const_intrinsic!(_mm512_cmp_ps_mask, _CMP_LT_OQ, self, rhs) as u64)
408    }
409    #[inline(always)]
410    unsafe fn cmp_f32_le(self, rhs: Self) -> Self::MaskType {
411        Avx512Mask(execute_const_intrinsic!(_mm512_cmp_ps_mask, _CMP_LE_OQ, self, rhs) as u64)
412    }
413    #[inline(always)]
414    unsafe fn cmp_f32_gt(self, rhs: Self) -> Self::MaskType {
415        Avx512Mask(execute_const_intrinsic!(_mm512_cmp_ps_mask, _CMP_GT_OQ, self, rhs) as u64)
416    }
417    #[inline(always)]
418    unsafe fn cmp_f32_ge(self, rhs: Self) -> Self::MaskType {
419        Avx512Mask(execute_const_intrinsic!(_mm512_cmp_ps_mask, _CMP_GE_OQ, self, rhs) as u64)
420    }
421    #[inline(always)]
422    unsafe fn cmp_f32_neq(self, rhs: Self) -> Self::MaskType {
423        Avx512Mask(execute_const_intrinsic!(_mm512_cmp_ps_mask, _CMP_NEQ_OQ, self, rhs) as u64)
424    }
425    #[inline(always)]
426    unsafe fn cmp_i64_eq(self, rhs: Self) -> Self::MaskType {
427        Avx512Mask(execute_intrinsic!(_mm512_cmpeq_epi64_mask, self, rhs) as u64)
428    }
429    #[inline(always)]
430    unsafe fn cmp_i64_gt(self, rhs: Self) -> Self::MaskType {
431        Avx512Mask(execute_intrinsic!(_mm512_cmpgt_epi64_mask, self, rhs) as u64)
432    }
433    #[inline(always)]
434    unsafe fn cmp_i32_eq(self, rhs: Self) -> Self::MaskType {
435        Avx512Mask(execute_intrinsic!(_mm512_cmpeq_epi32_mask, self, rhs) as u64)
436    }
437    #[inline(always)]
438    unsafe fn cmp_i32_gt(self, rhs: Self) -> Self::MaskType {
439        Avx512Mask(execute_intrinsic!(_mm512_cmpgt_epi32_mask, self, rhs) as u64)
440    }
441    #[inline(always)]
442    unsafe fn cmp_i16_eq(self, rhs: Self) -> Self::MaskType {
443        Avx512Mask(execute_intrinsic!(_mm512_cmpeq_epi16_mask, self, rhs) as u64)
444    }
445    #[inline(always)]
446    unsafe fn cmp_i16_gt(self, rhs: Self) -> Self::MaskType {
447        Avx512Mask(execute_intrinsic!(_mm512_cmpgt_epi16_mask, self, rhs) as u64)
448    }
449    #[inline(always)]
450    unsafe fn cmp_i8_eq(self, rhs: Self) -> Self::MaskType {
451        Avx512Mask(execute_intrinsic!(_mm512_cmpeq_epi8_mask, self, rhs) as u64)
452    }
453    #[inline(always)]
454    unsafe fn cmp_i8_gt(self, rhs: Self) -> Self::MaskType {
455        Avx512Mask(execute_intrinsic!(_mm512_cmpgt_epi8_mask, self, rhs) as u64)
456    }
457
458    #[inline(always)]
459    unsafe fn max_f64(self, rhs: Self) -> Self {
460        self_from_op!(_mm512_max_pd, self, rhs)
461    }
462    #[inline(always)]
463    unsafe fn min_f64(self, rhs: Self) -> Self {
464        self_from_op!(_mm512_min_pd, self, rhs)
465    }
466    #[inline(always)]
467    unsafe fn max_f32(self, rhs: Self) -> Self {
468        self_from_op!(_mm512_max_ps, self, rhs)
469    }
470    #[inline(always)]
471    unsafe fn min_f32(self, rhs: Self) -> Self {
472        self_from_op!(_mm512_min_ps, self, rhs)
473    }
474    #[inline(always)]
475    unsafe fn max_i32(self, rhs: Self) -> Self {
476        self_from_op!(_mm512_max_epi32, self, rhs)
477    }
478    #[inline(always)]
479    unsafe fn min_i32(self, rhs: Self) -> Self {
480        self_from_op!(_mm512_min_epi32, self, rhs)
481    }
482    #[inline(always)]
483    unsafe fn max_i16(self, rhs: Self) -> Self {
484        self_from_op!(_mm512_max_epi16, self, rhs)
485    }
486    #[inline(always)]
487    unsafe fn min_i16(self, rhs: Self) -> Self {
488        self_from_op!(_mm512_min_epi16, self, rhs)
489    }
490    #[inline(always)]
491    unsafe fn max_i8(self, rhs: Self) -> Self {
492        self_from_op!(_mm512_max_epi8, self, rhs)
493    }
494    #[inline(always)]
495    unsafe fn min_i8(self, rhs: Self) -> Self {
496        self_from_op!(_mm512_min_epi8, self, rhs)
497    }
498    #[inline(always)]
499    unsafe fn max_u32(self, rhs: Self) -> Self {
500        self_from_op!(_mm512_max_epu32, self, rhs)
501    }
502    #[inline(always)]
503    unsafe fn min_u32(self, rhs: Self) -> Self {
504        self_from_op!(_mm512_min_epu32, self, rhs)
505    }
506    #[inline(always)]
507    unsafe fn max_u16(self, rhs: Self) -> Self {
508        self_from_op!(_mm512_max_epu16, self, rhs)
509    }
510    #[inline(always)]
511    unsafe fn min_u16(self, rhs: Self) -> Self {
512        self_from_op!(_mm512_min_epu16, self, rhs)
513    }
514    #[inline(always)]
515    unsafe fn max_u8(self, rhs: Self) -> Self {
516        self_from_op!(_mm512_max_epu8, self, rhs)
517    }
518    #[inline(always)]
519    unsafe fn min_u8(self, rhs: Self) -> Self {
520        self_from_op!(_mm512_min_epu8, self, rhs)
521    }
522}
523
524// TODO: Make a custom trait for handling this transmutation into i*.
525impl SimdSplatImpl for Avx512Reg {
526    #[inline(always)]
527    unsafe fn splat_64<T>(val: T) -> Self {
528        self_from_op!(_mm512_set1_epi64, val)
529    }
530    #[inline(always)]
531    unsafe fn splat_32<T>(val: T) -> Self {
532        self_from_op!(_mm512_set1_epi32, val)
533    }
534    #[inline(always)]
535    unsafe fn splat_16<T>(val: T) -> Self {
536        self_from_op!(_mm512_set1_epi16, val)
537    }
538    #[inline(always)]
539    unsafe fn splat_8<T>(val: T) -> Self {
540        self_from_op!(_mm512_set1_epi8, val)
541    }
542}
543
544impl SimdBitwiseImpl for Avx512Mask {
545    #[inline(always)]
546    unsafe fn and(self, rhs: Self) -> Self {
547        self_from_op!(_kand_mask64, self, rhs)
548    }
549    #[inline(always)]
550    unsafe fn or(self, rhs: Self) -> Self {
551        self_from_op!(_kor_mask64, self, rhs)
552    }
553    #[inline(always)]
554    unsafe fn xor(self, rhs: Self) -> Self {
555        self_from_op!(_kxor_mask64, self, rhs)
556    }
557    #[inline(always)]
558    unsafe fn not(self) -> Self {
559        self_from_op!(_knot_mask64, self)
560    }
561    #[inline(always)]
562    unsafe fn and_not(self, rhs: Self) -> Self {
563        self_from_op!(_kandn_mask64, rhs, self)
564    }
565}
566
567impl SimdGatherImpl for Avx512Reg {
568    #[inline(always)]
569    unsafe fn gather_32_from_32<T, const B: i32>(self, ptr: *const T) -> Self {
570        self_from_const_op!(_mm512_i32gather_epi32, B, self, ptr)
571    }
572    // #[inline(always)] unsafe fn gather_64_from_32<T, const B: i32>(self, ptr: *const T) -> Self { self_from_const_op!(_mm512_i32gather_epi64, B, ptr, self) }
573    // #[inline(always)] unsafe fn gather_32_from_64<T, const B: i32>(self, ptr: *const T) -> Self { self_from_const_op!(_mm512_i64gather_epi32, B, ptr, self) }
574    #[inline(always)]
575    unsafe fn gather_64_from_64<T, const B: i32>(self, ptr: *const T) -> Self {
576        self_from_const_op!(_mm512_i64gather_epi64, B, self, ptr)
577    }
578}
579
580impl SimdSqrtImpl for Avx512Reg {
581    #[inline(always)]
582    unsafe fn sqrt_f64(self) -> Self {
583        self_from_op!(_mm512_sqrt_pd, self)
584    }
585    #[inline(always)]
586    unsafe fn sqrt_f32(self) -> Self {
587        self_from_op!(_mm512_sqrt_ps, self)
588    }
589    #[inline(always)]
590    unsafe fn rsqrt_f32(self) -> Self {
591        self_from_op!(_mm512_rsqrt14_ps, self)
592    }
593}
594
595impl SimdAllBitsImpl for Avx512Mask {
596    #[inline(always)]
597    unsafe fn all_zero(self) -> bool {
598        self.0 == 0
599    }
600}
601
602impl SimdNegateImpl for Avx512Reg {
603    #[inline(always)]
604    unsafe fn negate_f64(self) -> Self {
605        unsafe { Self::splat_64(-0.0f64).xor(self) }
606    }
607    #[inline(always)]
608    unsafe fn negate_f32(self) -> Self {
609        unsafe { Self::splat_32(-0.0f64).xor(self) }
610    }
611}
612
613impl SimdBlockShiftImpl for Avx512Reg {
614    #[inline(always)]
615    unsafe fn block_left_byte_shift<const N: i32>(self) -> Self {
616        self_from_const_op!(_mm512_bslli_epi128, N, self)
617    }
618    #[inline(always)]
619    unsafe fn block_right_byte_shift<const N: i32>(self) -> Self {
620        self_from_const_op!(_mm512_bsrli_epi128, N, self)
621    }
622}
623
624impl SimdMaskBitConversion for Avx512Mask {
625    #[inline(always)]
626    unsafe fn to_bits_64(self) -> u64 {
627        self.0
628    }
629    #[inline(always)]
630    unsafe fn to_bits_32(self) -> u64 {
631        self.0
632    }
633    #[inline(always)]
634    unsafe fn to_bits_8(self) -> u64 {
635        self.0
636    }
637    #[inline(always)]
638    unsafe fn from_bits_64(bitmask: u64) -> Self {
639        unsafe { transmute(bitmask) }
640    }
641    #[inline(always)]
642    unsafe fn from_bits_32(bitmask: u64) -> Self {
643        unsafe { transmute(bitmask) }
644    }
645    #[inline(always)]
646    unsafe fn from_bits_16(bitmask: u64) -> Self {
647        unsafe { transmute(bitmask) }
648    }
649    #[inline(always)]
650    unsafe fn from_bits_8(bitmask: u64) -> Self {
651        unsafe { transmute(bitmask) }
652    }
653}
654
655impl SimdLaneShiftImpl for Avx512Reg {
656    #[inline(always)]
657    unsafe fn right_lane_shift_32(self, n: u32) -> Self {
658        match n {
659            0 => self,
660            1 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 15) }),
661            2 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 14) }),
662            3 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 13) }),
663            4 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 12) }),
664            5 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 11) }),
665            6 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 10) }),
666            7 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 9) }),
667            8 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 8) }),
668            9 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 7) }),
669            10 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 6) }),
670            11 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 5) }),
671            12 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 4) }),
672            13 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 3) }),
673            14 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 2) }),
674            15 => Self(unsafe { _mm512_alignr_epi32(self.0, _mm512_setzero_si512(), 1) }),
675            _ => unsafe { Self::zero() },
676        }
677    }
678
679    #[inline(always)]
680    unsafe fn left_lane_shift_32(self, n: u32) -> Self {
681        match n {
682            0 => self,
683            1 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 1) }),
684            2 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 2) }),
685            3 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 3) }),
686            4 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 4) }),
687            5 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 5) }),
688            6 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 6) }),
689            7 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 7) }),
690            8 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 8) }),
691            9 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 9) }),
692            10 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 10) }),
693            11 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 11) }),
694            12 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 12) }),
695            13 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 13) }),
696            14 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 14) }),
697            15 => Self(unsafe { _mm512_alignr_epi32(_mm512_setzero_si512(), self.0, 15) }),
698            _ => unsafe { Self::zero() },
699        }
700    }
701}
702