Skip to main content

simply_simd/architectures/intrinsics/
avx2.rs

1use std::arch::x86_64::*;
2use std::mem::{transmute, transmute_copy};
3
4use crate::architectures::interface::*;
5use crate::architectures::macros::*;
6
7#[derive(Copy, Clone)]
8#[repr(transparent)]
9pub struct Avx2Reg(pub __m256i);
10impl SimdArch for Avx2Reg {}
11impl MaskArch for Avx2Reg {}
12
13impl SimdAddImpl for Avx2Reg {
14    #[inline(always)]
15    unsafe fn f64_add(self, rhs: Self) -> Self {
16        self_from_op!(_mm256_add_pd, self, rhs)
17    }
18    #[inline(always)]
19    unsafe fn f32_add(self, rhs: Self) -> Self {
20        self_from_op!(_mm256_add_ps, self, rhs)
21    }
22    #[inline(always)]
23    unsafe fn i64_add(self, rhs: Self) -> Self {
24        self_from_op!(_mm256_add_epi64, self, rhs)
25    }
26    #[inline(always)]
27    unsafe fn i32_add(self, rhs: Self) -> Self {
28        self_from_op!(_mm256_add_epi32, self, rhs)
29    }
30    #[inline(always)]
31    unsafe fn i16_add(self, rhs: Self) -> Self {
32        self_from_op!(_mm256_add_epi16, self, rhs)
33    }
34    #[inline(always)]
35    unsafe fn i8_add(self, rhs: Self) -> Self {
36        self_from_op!(_mm256_add_epi8, self, rhs)
37    }
38}
39
40impl SimdSubImpl for Avx2Reg {
41    #[inline(always)]
42    unsafe fn f64_sub(self, rhs: Self) -> Self {
43        self_from_op!(_mm256_sub_pd, self, rhs)
44    }
45    #[inline(always)]
46    unsafe fn f32_sub(self, rhs: Self) -> Self {
47        self_from_op!(_mm256_sub_ps, self, rhs)
48    }
49    #[inline(always)]
50    unsafe fn i64_sub(self, rhs: Self) -> Self {
51        self_from_op!(_mm256_sub_epi64, self, rhs)
52    }
53    #[inline(always)]
54    unsafe fn i32_sub(self, rhs: Self) -> Self {
55        self_from_op!(_mm256_sub_epi32, self, rhs)
56    }
57    #[inline(always)]
58    unsafe fn i16_sub(self, rhs: Self) -> Self {
59        self_from_op!(_mm256_sub_epi16, self, rhs)
60    }
61    #[inline(always)]
62    unsafe fn i8_sub(self, rhs: Self) -> Self {
63        self_from_op!(_mm256_sub_epi8, self, rhs)
64    }
65}
66
67impl SimdMulImpl for Avx2Reg {
68    #[inline(always)]
69    unsafe fn f64_mul(self, rhs: Self) -> Self {
70        self_from_op!(_mm256_mul_pd, self, rhs)
71    }
72    #[inline(always)]
73    unsafe fn f32_mul(self, rhs: Self) -> Self {
74        self_from_op!(_mm256_mul_ps, self, rhs)
75    }
76    #[inline(always)]
77    unsafe fn i32_mul(self, rhs: Self) -> Self {
78        self_from_op!(_mm256_mullo_epi32, self, rhs)
79    }
80    #[inline(always)]
81    unsafe fn i16_mul(self, rhs: Self) -> Self {
82        self_from_op!(_mm256_mullo_epi16, self, rhs)
83    }
84}
85
86impl SimdDivImpl for Avx2Reg {
87    #[inline(always)]
88    unsafe fn f64_div(self, rhs: Self) -> Self {
89        self_from_op!(_mm256_div_pd, self, rhs)
90    }
91    #[inline(always)]
92    unsafe fn f32_div(self, rhs: Self) -> Self {
93        self_from_op!(_mm256_div_ps, self, rhs)
94    }
95}
96
97impl SimdBitwiseImpl for Avx2Reg {
98    #[inline(always)]
99    unsafe fn and(self, rhs: Self) -> Self {
100        self_from_op!(_mm256_and_si256, self, rhs)
101    }
102    #[inline(always)]
103    unsafe fn or(self, rhs: Self) -> Self {
104        self_from_op!(_mm256_or_si256, self, rhs)
105    }
106    #[inline(always)]
107    unsafe fn xor(self, rhs: Self) -> Self {
108        self_from_op!(_mm256_xor_si256, self, rhs)
109    }
110    #[inline(always)]
111    unsafe fn not(self) -> Self {
112        unsafe { Self(self.xor(Self::splat_32(!0)).0) }
113    }
114    #[inline(always)]
115    unsafe fn and_not(self, rhs: Self) -> Self {
116        self_from_op!(_mm256_andnot_si256, rhs, self)
117    }
118}
119
120impl SimdShiftImpl for Avx2Reg {
121    #[inline(always)]
122    unsafe fn sllv_64(self, rhs: Self) -> Self {
123        self_from_op!(_mm256_sllv_epi64, self, rhs)
124    }
125    #[inline(always)]
126    unsafe fn srlv_64(self, rhs: Self) -> Self {
127        self_from_op!(_mm256_srlv_epi64, self, rhs)
128    }
129    #[inline(always)]
130    unsafe fn srav_64(self, rhs: Self) -> Self {
131        self_from_op!(_mm256_srav_epi64, self, rhs)
132    }
133    #[inline(always)]
134    unsafe fn sllv_32(self, rhs: Self) -> Self {
135        self_from_op!(_mm256_sllv_epi32, self, rhs)
136    }
137    #[inline(always)]
138    unsafe fn srlv_32(self, rhs: Self) -> Self {
139        self_from_op!(_mm256_srlv_epi32, self, rhs)
140    }
141    #[inline(always)]
142    unsafe fn srav_32(self, rhs: Self) -> Self {
143        self_from_op!(_mm256_srav_epi32, self, rhs)
144    }
145    #[inline(always)]
146    unsafe fn sllv_16(self, rhs: Self) -> Self {
147        self_from_op!(_mm256_sllv_epi16, self, rhs)
148    }
149    #[inline(always)]
150    unsafe fn srlv_16(self, rhs: Self) -> Self {
151        self_from_op!(_mm256_srlv_epi16, self, rhs)
152    }
153    #[inline(always)]
154    unsafe fn srav_16(self, rhs: Self) -> Self {
155        self_from_op!(_mm256_srav_epi16, self, rhs)
156    }
157}
158
159impl SimdLoadImpl for Avx2Reg {
160    type MaskType = Self;
161    #[inline(always)]
162    unsafe fn load_aligned<T>(ptr: *const T) -> Self {
163        self_from_op!(_mm256_load_si256, ptr)
164    }
165    #[inline(always)]
166    unsafe fn load_unaligned<T>(ptr: *const T) -> Self {
167        self_from_op!(_mm256_loadu_si256, ptr)
168    }
169    #[inline(always)]
170    unsafe fn masked_load_64<T>(ptr: *const T, mask: Self::MaskType) -> Self {
171        self_from_op!(_mm256_maskload_epi64, ptr, mask)
172    }
173    #[inline(always)]
174    unsafe fn masked_load_32<T>(ptr: *const T, mask: Self::MaskType) -> Self {
175        self_from_op!(_mm256_maskload_epi32, ptr, mask)
176    }
177}
178
179impl SimdStoreImpl for Avx2Reg {
180    type MaskType = Self;
181    #[inline(always)]
182    unsafe fn store_aligned<T>(self, ptr: *mut T) {
183        execute_intrinsic!(_mm256_store_si256, ptr, self);
184    }
185    #[inline(always)]
186    unsafe fn store_unaligned<T>(self, ptr: *mut T) {
187        execute_intrinsic!(_mm256_storeu_si256, ptr, self);
188    }
189    #[inline(always)]
190    unsafe fn masked_store_64<T>(self, ptr: *mut T, mask: Self::MaskType) {
191        execute_intrinsic!(_mm256_maskstore_epi64, ptr, mask, self);
192    }
193    #[inline(always)]
194    unsafe fn masked_store_32<T>(self, ptr: *mut T, mask: Self::MaskType) {
195        execute_intrinsic!(_mm256_maskstore_epi32, ptr, mask, self);
196    }
197}
198
199impl SimdZeroImpl for Avx2Reg {
200    #[inline(always)]
201    unsafe fn zero() -> Self {
202        self_from_op!(_mm256_setzero_si256,)
203    }
204}
205
206impl SimdFloatCastsImpl for Avx2Reg {
207    #[inline(always)]
208    unsafe fn float_to_int_trunc(self) -> Self {
209        self_from_op!(_mm256_cvttps_epi32, self)
210    }
211    #[inline(always)]
212    unsafe fn float_to_int_round(self) -> Self {
213        self_from_op!(_mm256_cvtps_epi32, self)
214    }
215}
216
217impl SimdIntCastsImpl for Avx2Reg {
218    #[inline(always)]
219    unsafe fn int_to_float(self) -> Self {
220        self_from_op!(_mm256_cvtepi32_ps, self)
221    }
222}
223
224impl SimdPermuteImpl for Avx2Reg {
225    #[inline(always)]
226    unsafe fn permute_32(self, rhs: Self) -> Self {
227        self_from_op!(_mm256_permutevar8x32_epi32, self, rhs)
228    }
229    #[inline(always)]
230    unsafe fn permute_8(self, rhs: Self) -> Self {
231        self_from_op!(_mm256_shuffle_epi8, self, rhs)
232    }
233}
234
235impl SimdVariableBlendImpl for Avx2Reg {
236    type VecType = Self;
237    #[inline(always)]
238    unsafe fn vblend_64(
239        self,
240        true_values: Self::VecType,
241        false_values: Self::VecType,
242    ) -> Self::VecType {
243        self_from_op!(_mm256_blendv_pd, false_values, true_values, self)
244    }
245    #[inline(always)]
246    unsafe fn vblend_32(
247        self,
248        true_values: Self::VecType,
249        false_values: Self::VecType,
250    ) -> Self::VecType {
251        self_from_op!(_mm256_blendv_ps, false_values, true_values, self)
252    }
253    #[inline(always)]
254    unsafe fn vblend_8(
255        self,
256        true_values: Self::VecType,
257        false_values: Self::VecType,
258    ) -> Self::VecType {
259        self_from_op!(_mm256_blendv_epi8, false_values, true_values, self)
260    }
261}
262
263impl SimdImmediateBlendImpl for Avx2Reg {
264    #[inline(always)]
265    unsafe fn blend_64<const N: i32>(self, false_values: Self) -> Self {
266        self_from_const_op!(_mm256_blend_pd, N, false_values, self)
267    }
268    #[inline(always)]
269    unsafe fn blend_32<const N: i32>(self, false_values: Self) -> Self {
270        self_from_const_op!(_mm256_blend_ps, N, false_values, self)
271    }
272}
273
274impl SimdMulAddImpl for Avx2Reg {
275    #[inline(always)]
276    unsafe fn mul_add_f64(self, mult: Self, add: Self) -> Self {
277        self_from_op!(_mm256_fmadd_pd, self, mult, add)
278    }
279    #[inline(always)]
280    unsafe fn mul_sub_f64(self, mult: Self, sub: Self) -> Self {
281        self_from_op!(_mm256_fmsub_pd, self, mult, sub)
282    }
283    #[inline(always)]
284    unsafe fn negated_mul_add_f64(self, mult: Self, add: Self) -> Self {
285        self_from_op!(_mm256_fnmadd_pd, self, mult, add)
286    }
287    #[inline(always)]
288    unsafe fn negated_mul_sub_f64(self, mult: Self, sub: Self) -> Self {
289        self_from_op!(_mm256_fnmsub_pd, self, mult, sub)
290    }
291    #[inline(always)]
292    unsafe fn mul_add_f32(self, mult: Self, add: Self) -> Self {
293        self_from_op!(_mm256_fmadd_ps, self, mult, add)
294    }
295    #[inline(always)]
296    unsafe fn mul_sub_f32(self, mult: Self, sub: Self) -> Self {
297        self_from_op!(_mm256_fmsub_ps, self, mult, sub)
298    }
299    #[inline(always)]
300    unsafe fn negated_mul_add_f32(self, mult: Self, add: Self) -> Self {
301        self_from_op!(_mm256_fnmadd_ps, self, mult, add)
302    }
303    #[inline(always)]
304    unsafe fn negated_mul_sub_f32(self, mult: Self, sub: Self) -> Self {
305        self_from_op!(_mm256_fnmsub_ps, self, mult, sub)
306    }
307}
308
309impl SimdRoundImpl for Avx2Reg {
310    #[inline(always)]
311    unsafe fn round_f64(self) -> Self {
312        self_from_const_op!(
313            _mm256_round_pd,
314            _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC,
315            self
316        )
317    }
318    #[inline(always)]
319    unsafe fn round_f32(self) -> Self {
320        self_from_const_op!(
321            _mm256_round_ps,
322            _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC,
323            self
324        )
325    }
326    #[inline(always)]
327    unsafe fn floor_f64(self) -> Self {
328        self_from_const_op!(
329            _mm256_round_pd,
330            _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC,
331            self
332        )
333    }
334    #[inline(always)]
335    unsafe fn floor_f32(self) -> Self {
336        self_from_const_op!(
337            _mm256_round_ps,
338            _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC,
339            self
340        )
341    }
342    #[inline(always)]
343    unsafe fn ceil_f64(self) -> Self {
344        self_from_const_op!(
345            _mm256_round_pd,
346            _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC,
347            self
348        )
349    }
350    #[inline(always)]
351    unsafe fn ceil_f32(self) -> Self {
352        self_from_const_op!(
353            _mm256_round_ps,
354            _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC,
355            self
356        )
357    }
358}
359
360impl SimdPartialOrdImpl for Avx2Reg {
361    type MaskType = Self;
362    #[inline(always)]
363    unsafe fn cmp_f64_eq(self, rhs: Self) -> Self {
364        self_from_const_op!(_mm256_cmp_pd, _CMP_EQ_OQ, self, rhs)
365    }
366    #[inline(always)]
367    unsafe fn cmp_f64_lt(self, rhs: Self) -> Self {
368        self_from_const_op!(_mm256_cmp_pd, _CMP_LT_OQ, self, rhs)
369    }
370    #[inline(always)]
371    unsafe fn cmp_f64_le(self, rhs: Self) -> Self {
372        self_from_const_op!(_mm256_cmp_pd, _CMP_LE_OQ, self, rhs)
373    }
374    #[inline(always)]
375    unsafe fn cmp_f64_gt(self, rhs: Self) -> Self {
376        self_from_const_op!(_mm256_cmp_pd, _CMP_GT_OQ, self, rhs)
377    }
378    #[inline(always)]
379    unsafe fn cmp_f64_ge(self, rhs: Self) -> Self {
380        self_from_const_op!(_mm256_cmp_pd, _CMP_GE_OQ, self, rhs)
381    }
382    #[inline(always)]
383    unsafe fn cmp_f64_neq(self, rhs: Self) -> Self {
384        self_from_const_op!(_mm256_cmp_pd, _CMP_NEQ_OQ, self, rhs)
385    }
386    #[inline(always)]
387    unsafe fn cmp_f32_eq(self, rhs: Self) -> Self {
388        self_from_const_op!(_mm256_cmp_ps, _CMP_EQ_OQ, self, rhs)
389    }
390    #[inline(always)]
391    unsafe fn cmp_f32_lt(self, rhs: Self) -> Self {
392        self_from_const_op!(_mm256_cmp_ps, _CMP_LT_OQ, self, rhs)
393    }
394    #[inline(always)]
395    unsafe fn cmp_f32_le(self, rhs: Self) -> Self {
396        self_from_const_op!(_mm256_cmp_ps, _CMP_LE_OQ, self, rhs)
397    }
398    #[inline(always)]
399    unsafe fn cmp_f32_gt(self, rhs: Self) -> Self {
400        self_from_const_op!(_mm256_cmp_ps, _CMP_GT_OQ, self, rhs)
401    }
402    #[inline(always)]
403    unsafe fn cmp_f32_ge(self, rhs: Self) -> Self {
404        self_from_const_op!(_mm256_cmp_ps, _CMP_GE_OQ, self, rhs)
405    }
406    #[inline(always)]
407    unsafe fn cmp_f32_neq(self, rhs: Self) -> Self {
408        self_from_const_op!(_mm256_cmp_ps, _CMP_NEQ_OQ, self, rhs)
409    }
410    #[inline(always)]
411    unsafe fn cmp_i64_eq(self, rhs: Self) -> Self {
412        self_from_op!(_mm256_cmpeq_epi64, self, rhs)
413    }
414    #[inline(always)]
415    unsafe fn cmp_i64_gt(self, rhs: Self) -> Self {
416        self_from_op!(_mm256_cmpgt_epi64, self, rhs)
417    }
418    #[inline(always)]
419    unsafe fn cmp_i32_eq(self, rhs: Self) -> Self {
420        self_from_op!(_mm256_cmpeq_epi32, self, rhs)
421    }
422    #[inline(always)]
423    unsafe fn cmp_i32_gt(self, rhs: Self) -> Self {
424        self_from_op!(_mm256_cmpgt_epi32, self, rhs)
425    }
426    #[inline(always)]
427    unsafe fn cmp_i16_eq(self, rhs: Self) -> Self {
428        self_from_op!(_mm256_cmpeq_epi16, self, rhs)
429    }
430    #[inline(always)]
431    unsafe fn cmp_i16_gt(self, rhs: Self) -> Self {
432        self_from_op!(_mm256_cmpgt_epi16, self, rhs)
433    }
434    #[inline(always)]
435    unsafe fn cmp_i8_eq(self, rhs: Self) -> Self {
436        self_from_op!(_mm256_cmpeq_epi8, self, rhs)
437    }
438    #[inline(always)]
439    unsafe fn cmp_i8_gt(self, rhs: Self) -> Self {
440        self_from_op!(_mm256_cmpgt_epi8, self, rhs)
441    }
442
443    #[inline(always)]
444    unsafe fn max_f64(self, rhs: Self) -> Self {
445        self_from_op!(_mm256_max_pd, self, rhs)
446    }
447    #[inline(always)]
448    unsafe fn min_f64(self, rhs: Self) -> Self {
449        self_from_op!(_mm256_min_pd, self, rhs)
450    }
451    #[inline(always)]
452    unsafe fn max_f32(self, rhs: Self) -> Self {
453        self_from_op!(_mm256_max_ps, self, rhs)
454    }
455    #[inline(always)]
456    unsafe fn min_f32(self, rhs: Self) -> Self {
457        self_from_op!(_mm256_min_ps, self, rhs)
458    }
459    #[inline(always)]
460    unsafe fn max_i32(self, rhs: Self) -> Self {
461        self_from_op!(_mm256_max_epi32, self, rhs)
462    }
463    #[inline(always)]
464    unsafe fn min_i32(self, rhs: Self) -> Self {
465        self_from_op!(_mm256_min_epi32, self, rhs)
466    }
467    #[inline(always)]
468    unsafe fn max_i16(self, rhs: Self) -> Self {
469        self_from_op!(_mm256_max_epi16, self, rhs)
470    }
471    #[inline(always)]
472    unsafe fn min_i16(self, rhs: Self) -> Self {
473        self_from_op!(_mm256_min_epi16, self, rhs)
474    }
475    #[inline(always)]
476    unsafe fn max_i8(self, rhs: Self) -> Self {
477        self_from_op!(_mm256_max_epi8, self, rhs)
478    }
479    #[inline(always)]
480    unsafe fn min_i8(self, rhs: Self) -> Self {
481        self_from_op!(_mm256_min_epi8, self, rhs)
482    }
483    #[inline(always)]
484    unsafe fn max_u32(self, rhs: Self) -> Self {
485        self_from_op!(_mm256_max_epu32, self, rhs)
486    }
487    #[inline(always)]
488    unsafe fn min_u32(self, rhs: Self) -> Self {
489        self_from_op!(_mm256_min_epu32, self, rhs)
490    }
491    #[inline(always)]
492    unsafe fn max_u16(self, rhs: Self) -> Self {
493        self_from_op!(_mm256_max_epu16, self, rhs)
494    }
495    #[inline(always)]
496    unsafe fn min_u16(self, rhs: Self) -> Self {
497        self_from_op!(_mm256_min_epu16, self, rhs)
498    }
499    #[inline(always)]
500    unsafe fn max_u8(self, rhs: Self) -> Self {
501        self_from_op!(_mm256_max_epu8, self, rhs)
502    }
503    #[inline(always)]
504    unsafe fn min_u8(self, rhs: Self) -> Self {
505        self_from_op!(_mm256_min_epu8, self, rhs)
506    }
507}
508
509// TODO: Make a custom trait for handling this transmutation into i*.
510impl SimdSplatImpl for Avx2Reg {
511    #[inline(always)]
512    unsafe fn splat_64<T>(val: T) -> Self {
513        self_from_op!(_mm256_set1_epi64x, val)
514    }
515    #[inline(always)]
516    unsafe fn splat_32<T>(val: T) -> Self {
517        self_from_op!(_mm256_set1_epi32, val)
518    }
519    #[inline(always)]
520    unsafe fn splat_16<T>(val: T) -> Self {
521        self_from_op!(_mm256_set1_epi16, val)
522    }
523    #[inline(always)]
524    unsafe fn splat_8<T>(val: T) -> Self {
525        self_from_op!(_mm256_set1_epi8, val)
526    }
527}
528
529impl SimdGatherImpl for Avx2Reg {
530    #[inline(always)]
531    unsafe fn gather_32_from_32<T, const B: i32>(self, ptr: *const T) -> Self {
532        self_from_const_op!(_mm256_i32gather_epi32, B, ptr, self)
533    }
534    // #[inline(always)] unsafe fn gather_64_from_32<T, const B: i32>(self, ptr: *const T) -> Self { self_from_const_op!(_mm256_i32gather_epi64, B, ptr, self) }
535    // #[inline(always)] unsafe fn gather_32_from_64<T, const B: i32>(self, ptr: *const T) -> Self { self_from_const_op!(_mm256_i64gather_epi32, B, ptr, self) }
536    #[inline(always)]
537    unsafe fn gather_64_from_64<T, const B: i32>(self, ptr: *const T) -> Self {
538        self_from_const_op!(_mm256_i64gather_epi64, B, ptr, self)
539    }
540}
541
542impl SimdSqrtImpl for Avx2Reg {
543    #[inline(always)]
544    unsafe fn sqrt_f64(self) -> Self {
545        self_from_op!(_mm256_sqrt_pd, self)
546    }
547    #[inline(always)]
548    unsafe fn sqrt_f32(self) -> Self {
549        self_from_op!(_mm256_sqrt_ps, self)
550    }
551    #[inline(always)]
552    unsafe fn rsqrt_f32(self) -> Self {
553        self_from_op!(_mm256_rsqrt_ps, self)
554    }
555}
556
557impl SimdAllBitsImpl for Avx2Reg {
558    #[inline(always)]
559    unsafe fn all_zero(self) -> bool {
560        execute_intrinsic!(_mm256_testz_si256, self, self) != 0
561    }
562}
563
564impl SimdNegateImpl for Avx2Reg {
565    #[inline(always)]
566    unsafe fn negate_f64(self) -> Self {
567        unsafe { Self::splat_64(-0.0f64).xor(self) }
568    }
569    #[inline(always)]
570    unsafe fn negate_f32(self) -> Self {
571        unsafe { Self::splat_32(-0.0f64).xor(self) }
572    }
573}
574
575impl SimdBlockShiftImpl for Avx2Reg {
576    #[inline(always)]
577    unsafe fn block_left_byte_shift<const N: i32>(self) -> Self {
578        self_from_const_op!(_mm256_bslli_epi128, N, self)
579    }
580    #[inline(always)]
581    unsafe fn block_right_byte_shift<const N: i32>(self) -> Self {
582        self_from_const_op!(_mm256_bsrli_epi128, N, self)
583    }
584}
585
586impl SimdMaskBitConversion for Avx2Reg {
587    #[inline(always)]
588    unsafe fn to_bits_64(self) -> u64 {
589        execute_intrinsic!(_mm256_movemask_pd, self) as u64
590    }
591    #[inline(always)]
592    unsafe fn to_bits_32(self) -> u64 {
593        execute_intrinsic!(_mm256_movemask_ps, self) as u64
594    }
595    #[inline(always)]
596    unsafe fn to_bits_8(self) -> u64 {
597        execute_intrinsic!(_mm256_movemask_epi8, self) as u64
598    }
599    #[inline(always)]
600    unsafe fn from_bits_64(bitmask: u64) -> Self {
601        let mask = self_from_op!(_mm256_set_epi64x, 1, 2, 4, 8);
602        let bits = self_from_op!(_mm256_set1_epi64x, bitmask as i64);
603        self_from_op!(_mm256_cmpgt_epi64, bits.and(mask), Self::zero())
604    }
605    #[inline(always)]
606    unsafe fn from_bits_32(bitmask: u64) -> Self {
607        let mask = self_from_op!(_mm256_set_epi32, 1, 2, 4, 8, 16, 32, 64, 128);
608        let bits = self_from_op!(_mm256_set1_epi32, bitmask as u32);
609        self_from_op!(_mm256_cmpgt_epi32, bits.and(mask), Self::zero())
610    }
611    #[inline(always)]
612    unsafe fn from_bits_16(bitmask: u64) -> Self {
613        #[rustfmt::skip]
614        let mask = self_from_op!(_mm256_set_epi16, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, -32768);
615        let bits = self_from_op!(_mm256_set1_epi16, bitmask as i16);
616        unsafe { self_from_op!(_mm256_cmpeq_epi16, bits.and(mask), Self::zero()).not() }
617    }
618    #[inline(always)]
619    unsafe fn from_bits_8(bitmask: u64) -> Self {
620        let b1 = bitmask as i8;
621        let b2 = (bitmask >> 8) as i8;
622        let b3 = (bitmask >> 16) as i8;
623        let b4 = (bitmask >> 24) as i8;
624        #[rustfmt::skip]
625        let mask = self_from_op!(_mm256_set_epi8,
626            1, 2, 4, 8, 16, 32, 64, -128,
627            1, 2, 4, 8, 16, 32, 64, -128,
628            1, 2, 4, 8, 16, 32, 64, -128,
629            1, 2, 4, 8, 16, 32, 64, -128
630        );
631        #[rustfmt::skip]
632        let bits = self_from_op!(_mm256_set_epi8,
633            b1, b1, b1, b1, b1, b1, b1, b1,
634            b2, b2, b2, b2, b2, b2, b2, b2,
635            b3, b3, b3, b3, b3, b3, b3, b3,
636            b4, b4, b4, b4, b4, b4, b4, b4
637        );
638        unsafe { self_from_op!(_mm256_cmpeq_epi8, bits.and(mask), Self::zero()).not() }
639    }
640}
641
642impl SimdLaneShiftImpl for Avx2Reg {
643    #[inline(always)]
644    unsafe fn left_lane_shift_32(self, n: u32) -> Self {
645        unsafe {
646            match n {
647                0 => self,
648                #[rustfmt::skip]
649                1 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 1, 2, 3, 4, 5, 6, 7, 0)),
650                #[rustfmt::skip]
651                2 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 2, 3, 4, 5, 6, 7, 0, 0)),
652                #[rustfmt::skip]
653                3 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 3, 4, 5, 6, 7, 0, 0, 0)),
654                #[rustfmt::skip]
655                4 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 4, 5, 6, 7, 0, 0, 0, 0)),
656                #[rustfmt::skip]
657                5 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 5, 6, 7, 0, 0, 0, 0, 0)),
658                #[rustfmt::skip]
659                6 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 6, 7, 0, 0, 0, 0, 0, 0)),
660                #[rustfmt::skip]
661                7 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 7, 0, 0, 0, 0, 0, 0, 0)),
662                _ => Self::zero(),
663            }
664        }
665    }
666    #[inline(always)]
667    unsafe fn right_lane_shift_32(self, n: u32) -> Self {
668        unsafe {
669            match n {
670                0 => self,
671                1 => Self::zero()
672                    .blend_32::<0x80>(self)
673                    .permute_32(self_from_op!(_mm256_setr_epi32, 7, 0, 1, 2, 3, 4, 5, 6)),
674                2 => Self::zero()
675                    .blend_32::<0x80>(self)
676                    .permute_32(self_from_op!(_mm256_setr_epi32, 7, 7, 0, 1, 2, 3, 4, 5)),
677                3 => Self::zero()
678                    .blend_32::<0x80>(self)
679                    .permute_32(self_from_op!(_mm256_setr_epi32, 7, 7, 7, 0, 1, 2, 3, 4)),
680                4 => Self::zero()
681                    .blend_32::<0x80>(self)
682                    .permute_32(self_from_op!(_mm256_setr_epi32, 7, 7, 7, 7, 0, 1, 2, 3)),
683                5 => Self::zero()
684                    .blend_32::<0x80>(self)
685                    .permute_32(self_from_op!(_mm256_setr_epi32, 7, 7, 7, 7, 7, 0, 1, 2)),
686                6 => Self::zero()
687                    .blend_32::<0x80>(self)
688                    .permute_32(self_from_op!(_mm256_setr_epi32, 7, 7, 7, 7, 7, 7, 0, 1)),
689                7 => Self::zero()
690                    .blend_32::<0x80>(self)
691                    .permute_32(self_from_op!(_mm256_setr_epi32, 7, 7, 7, 7, 7, 7, 7, 0)),
692                _ => Self::zero(),
693            }
694        }
695    }
696}