1use std::arch::x86_64::*;
2use std::mem::{transmute, transmute_copy};
3
4use crate::architectures::interface::*;
5use crate::architectures::macros::*;
6
7#[derive(Copy, Clone)]
8#[repr(transparent)]
9pub struct Avx2Reg(pub __m256i);
10impl SimdArch for Avx2Reg {}
11impl MaskArch for Avx2Reg {}
12
13impl SimdAddImpl for Avx2Reg {
14 #[inline(always)]
15 unsafe fn f64_add(self, rhs: Self) -> Self {
16 self_from_op!(_mm256_add_pd, self, rhs)
17 }
18 #[inline(always)]
19 unsafe fn f32_add(self, rhs: Self) -> Self {
20 self_from_op!(_mm256_add_ps, self, rhs)
21 }
22 #[inline(always)]
23 unsafe fn i64_add(self, rhs: Self) -> Self {
24 self_from_op!(_mm256_add_epi64, self, rhs)
25 }
26 #[inline(always)]
27 unsafe fn i32_add(self, rhs: Self) -> Self {
28 self_from_op!(_mm256_add_epi32, self, rhs)
29 }
30 #[inline(always)]
31 unsafe fn i16_add(self, rhs: Self) -> Self {
32 self_from_op!(_mm256_add_epi16, self, rhs)
33 }
34 #[inline(always)]
35 unsafe fn i8_add(self, rhs: Self) -> Self {
36 self_from_op!(_mm256_add_epi8, self, rhs)
37 }
38}
39
40impl SimdSubImpl for Avx2Reg {
41 #[inline(always)]
42 unsafe fn f64_sub(self, rhs: Self) -> Self {
43 self_from_op!(_mm256_sub_pd, self, rhs)
44 }
45 #[inline(always)]
46 unsafe fn f32_sub(self, rhs: Self) -> Self {
47 self_from_op!(_mm256_sub_ps, self, rhs)
48 }
49 #[inline(always)]
50 unsafe fn i64_sub(self, rhs: Self) -> Self {
51 self_from_op!(_mm256_sub_epi64, self, rhs)
52 }
53 #[inline(always)]
54 unsafe fn i32_sub(self, rhs: Self) -> Self {
55 self_from_op!(_mm256_sub_epi32, self, rhs)
56 }
57 #[inline(always)]
58 unsafe fn i16_sub(self, rhs: Self) -> Self {
59 self_from_op!(_mm256_sub_epi16, self, rhs)
60 }
61 #[inline(always)]
62 unsafe fn i8_sub(self, rhs: Self) -> Self {
63 self_from_op!(_mm256_sub_epi8, self, rhs)
64 }
65}
66
67impl SimdMulImpl for Avx2Reg {
68 #[inline(always)]
69 unsafe fn f64_mul(self, rhs: Self) -> Self {
70 self_from_op!(_mm256_mul_pd, self, rhs)
71 }
72 #[inline(always)]
73 unsafe fn f32_mul(self, rhs: Self) -> Self {
74 self_from_op!(_mm256_mul_ps, self, rhs)
75 }
76 #[inline(always)]
77 unsafe fn i32_mul(self, rhs: Self) -> Self {
78 self_from_op!(_mm256_mullo_epi32, self, rhs)
79 }
80 #[inline(always)]
81 unsafe fn i16_mul(self, rhs: Self) -> Self {
82 self_from_op!(_mm256_mullo_epi16, self, rhs)
83 }
84}
85
86impl SimdDivImpl for Avx2Reg {
87 #[inline(always)]
88 unsafe fn f64_div(self, rhs: Self) -> Self {
89 self_from_op!(_mm256_div_pd, self, rhs)
90 }
91 #[inline(always)]
92 unsafe fn f32_div(self, rhs: Self) -> Self {
93 self_from_op!(_mm256_div_ps, self, rhs)
94 }
95}
96
97impl SimdBitwiseImpl for Avx2Reg {
98 #[inline(always)]
99 unsafe fn and(self, rhs: Self) -> Self {
100 self_from_op!(_mm256_and_si256, self, rhs)
101 }
102 #[inline(always)]
103 unsafe fn or(self, rhs: Self) -> Self {
104 self_from_op!(_mm256_or_si256, self, rhs)
105 }
106 #[inline(always)]
107 unsafe fn xor(self, rhs: Self) -> Self {
108 self_from_op!(_mm256_xor_si256, self, rhs)
109 }
110 #[inline(always)]
111 unsafe fn not(self) -> Self {
112 unsafe { Self(self.xor(Self::splat_32(!0)).0) }
113 }
114 #[inline(always)]
115 unsafe fn and_not(self, rhs: Self) -> Self {
116 self_from_op!(_mm256_andnot_si256, rhs, self)
117 }
118}
119
120impl SimdShiftImpl for Avx2Reg {
121 #[inline(always)]
122 unsafe fn sllv_64(self, rhs: Self) -> Self {
123 self_from_op!(_mm256_sllv_epi64, self, rhs)
124 }
125 #[inline(always)]
126 unsafe fn srlv_64(self, rhs: Self) -> Self {
127 self_from_op!(_mm256_srlv_epi64, self, rhs)
128 }
129 #[inline(always)]
130 unsafe fn srav_64(self, rhs: Self) -> Self {
131 self_from_op!(_mm256_srav_epi64, self, rhs)
132 }
133 #[inline(always)]
134 unsafe fn sllv_32(self, rhs: Self) -> Self {
135 self_from_op!(_mm256_sllv_epi32, self, rhs)
136 }
137 #[inline(always)]
138 unsafe fn srlv_32(self, rhs: Self) -> Self {
139 self_from_op!(_mm256_srlv_epi32, self, rhs)
140 }
141 #[inline(always)]
142 unsafe fn srav_32(self, rhs: Self) -> Self {
143 self_from_op!(_mm256_srav_epi32, self, rhs)
144 }
145 #[inline(always)]
146 unsafe fn sllv_16(self, rhs: Self) -> Self {
147 self_from_op!(_mm256_sllv_epi16, self, rhs)
148 }
149 #[inline(always)]
150 unsafe fn srlv_16(self, rhs: Self) -> Self {
151 self_from_op!(_mm256_srlv_epi16, self, rhs)
152 }
153 #[inline(always)]
154 unsafe fn srav_16(self, rhs: Self) -> Self {
155 self_from_op!(_mm256_srav_epi16, self, rhs)
156 }
157}
158
159impl SimdLoadImpl for Avx2Reg {
160 type MaskType = Self;
161 #[inline(always)]
162 unsafe fn load_aligned<T>(ptr: *const T) -> Self {
163 self_from_op!(_mm256_load_si256, ptr)
164 }
165 #[inline(always)]
166 unsafe fn load_unaligned<T>(ptr: *const T) -> Self {
167 self_from_op!(_mm256_loadu_si256, ptr)
168 }
169 #[inline(always)]
170 unsafe fn masked_load_64<T>(ptr: *const T, mask: Self::MaskType) -> Self {
171 self_from_op!(_mm256_maskload_epi64, ptr, mask)
172 }
173 #[inline(always)]
174 unsafe fn masked_load_32<T>(ptr: *const T, mask: Self::MaskType) -> Self {
175 self_from_op!(_mm256_maskload_epi32, ptr, mask)
176 }
177}
178
179impl SimdStoreImpl for Avx2Reg {
180 type MaskType = Self;
181 #[inline(always)]
182 unsafe fn store_aligned<T>(self, ptr: *mut T) {
183 execute_intrinsic!(_mm256_store_si256, ptr, self);
184 }
185 #[inline(always)]
186 unsafe fn store_unaligned<T>(self, ptr: *mut T) {
187 execute_intrinsic!(_mm256_storeu_si256, ptr, self);
188 }
189 #[inline(always)]
190 unsafe fn masked_store_64<T>(self, ptr: *mut T, mask: Self::MaskType) {
191 execute_intrinsic!(_mm256_maskstore_epi64, ptr, mask, self);
192 }
193 #[inline(always)]
194 unsafe fn masked_store_32<T>(self, ptr: *mut T, mask: Self::MaskType) {
195 execute_intrinsic!(_mm256_maskstore_epi32, ptr, mask, self);
196 }
197}
198
199impl SimdZeroImpl for Avx2Reg {
200 #[inline(always)]
201 unsafe fn zero() -> Self {
202 self_from_op!(_mm256_setzero_si256,)
203 }
204}
205
206impl SimdFloatCastsImpl for Avx2Reg {
207 #[inline(always)]
208 unsafe fn float_to_int_trunc(self) -> Self {
209 self_from_op!(_mm256_cvttps_epi32, self)
210 }
211 #[inline(always)]
212 unsafe fn float_to_int_round(self) -> Self {
213 self_from_op!(_mm256_cvtps_epi32, self)
214 }
215}
216
217impl SimdIntCastsImpl for Avx2Reg {
218 #[inline(always)]
219 unsafe fn int_to_float(self) -> Self {
220 self_from_op!(_mm256_cvtepi32_ps, self)
221 }
222}
223
224impl SimdPermuteImpl for Avx2Reg {
225 #[inline(always)]
226 unsafe fn permute_32(self, rhs: Self) -> Self {
227 self_from_op!(_mm256_permutevar8x32_epi32, self, rhs)
228 }
229 #[inline(always)]
230 unsafe fn permute_8(self, rhs: Self) -> Self {
231 self_from_op!(_mm256_shuffle_epi8, self, rhs)
232 }
233}
234
235impl SimdVariableBlendImpl for Avx2Reg {
236 type VecType = Self;
237 #[inline(always)]
238 unsafe fn vblend_64(
239 self,
240 true_values: Self::VecType,
241 false_values: Self::VecType,
242 ) -> Self::VecType {
243 self_from_op!(_mm256_blendv_pd, false_values, true_values, self)
244 }
245 #[inline(always)]
246 unsafe fn vblend_32(
247 self,
248 true_values: Self::VecType,
249 false_values: Self::VecType,
250 ) -> Self::VecType {
251 self_from_op!(_mm256_blendv_ps, false_values, true_values, self)
252 }
253 #[inline(always)]
254 unsafe fn vblend_8(
255 self,
256 true_values: Self::VecType,
257 false_values: Self::VecType,
258 ) -> Self::VecType {
259 self_from_op!(_mm256_blendv_epi8, false_values, true_values, self)
260 }
261}
262
263impl SimdImmediateBlendImpl for Avx2Reg {
264 #[inline(always)]
265 unsafe fn blend_64<const N: i32>(self, false_values: Self) -> Self {
266 self_from_const_op!(_mm256_blend_pd, N, false_values, self)
267 }
268 #[inline(always)]
269 unsafe fn blend_32<const N: i32>(self, false_values: Self) -> Self {
270 self_from_const_op!(_mm256_blend_ps, N, false_values, self)
271 }
272}
273
274impl SimdMulAddImpl for Avx2Reg {
275 #[inline(always)]
276 unsafe fn mul_add_f64(self, mult: Self, add: Self) -> Self {
277 self_from_op!(_mm256_fmadd_pd, self, mult, add)
278 }
279 #[inline(always)]
280 unsafe fn mul_sub_f64(self, mult: Self, sub: Self) -> Self {
281 self_from_op!(_mm256_fmsub_pd, self, mult, sub)
282 }
283 #[inline(always)]
284 unsafe fn negated_mul_add_f64(self, mult: Self, add: Self) -> Self {
285 self_from_op!(_mm256_fnmadd_pd, self, mult, add)
286 }
287 #[inline(always)]
288 unsafe fn negated_mul_sub_f64(self, mult: Self, sub: Self) -> Self {
289 self_from_op!(_mm256_fnmsub_pd, self, mult, sub)
290 }
291 #[inline(always)]
292 unsafe fn mul_add_f32(self, mult: Self, add: Self) -> Self {
293 self_from_op!(_mm256_fmadd_ps, self, mult, add)
294 }
295 #[inline(always)]
296 unsafe fn mul_sub_f32(self, mult: Self, sub: Self) -> Self {
297 self_from_op!(_mm256_fmsub_ps, self, mult, sub)
298 }
299 #[inline(always)]
300 unsafe fn negated_mul_add_f32(self, mult: Self, add: Self) -> Self {
301 self_from_op!(_mm256_fnmadd_ps, self, mult, add)
302 }
303 #[inline(always)]
304 unsafe fn negated_mul_sub_f32(self, mult: Self, sub: Self) -> Self {
305 self_from_op!(_mm256_fnmsub_ps, self, mult, sub)
306 }
307}
308
309impl SimdRoundImpl for Avx2Reg {
310 #[inline(always)]
311 unsafe fn round_f64(self) -> Self {
312 self_from_const_op!(
313 _mm256_round_pd,
314 _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC,
315 self
316 )
317 }
318 #[inline(always)]
319 unsafe fn round_f32(self) -> Self {
320 self_from_const_op!(
321 _mm256_round_ps,
322 _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC,
323 self
324 )
325 }
326 #[inline(always)]
327 unsafe fn floor_f64(self) -> Self {
328 self_from_const_op!(
329 _mm256_round_pd,
330 _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC,
331 self
332 )
333 }
334 #[inline(always)]
335 unsafe fn floor_f32(self) -> Self {
336 self_from_const_op!(
337 _mm256_round_ps,
338 _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC,
339 self
340 )
341 }
342 #[inline(always)]
343 unsafe fn ceil_f64(self) -> Self {
344 self_from_const_op!(
345 _mm256_round_pd,
346 _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC,
347 self
348 )
349 }
350 #[inline(always)]
351 unsafe fn ceil_f32(self) -> Self {
352 self_from_const_op!(
353 _mm256_round_ps,
354 _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC,
355 self
356 )
357 }
358}
359
360impl SimdPartialOrdImpl for Avx2Reg {
361 type MaskType = Self;
362 #[inline(always)]
363 unsafe fn cmp_f64_eq(self, rhs: Self) -> Self {
364 self_from_const_op!(_mm256_cmp_pd, _CMP_EQ_OQ, self, rhs)
365 }
366 #[inline(always)]
367 unsafe fn cmp_f64_lt(self, rhs: Self) -> Self {
368 self_from_const_op!(_mm256_cmp_pd, _CMP_LT_OQ, self, rhs)
369 }
370 #[inline(always)]
371 unsafe fn cmp_f64_le(self, rhs: Self) -> Self {
372 self_from_const_op!(_mm256_cmp_pd, _CMP_LE_OQ, self, rhs)
373 }
374 #[inline(always)]
375 unsafe fn cmp_f64_gt(self, rhs: Self) -> Self {
376 self_from_const_op!(_mm256_cmp_pd, _CMP_GT_OQ, self, rhs)
377 }
378 #[inline(always)]
379 unsafe fn cmp_f64_ge(self, rhs: Self) -> Self {
380 self_from_const_op!(_mm256_cmp_pd, _CMP_GE_OQ, self, rhs)
381 }
382 #[inline(always)]
383 unsafe fn cmp_f64_neq(self, rhs: Self) -> Self {
384 self_from_const_op!(_mm256_cmp_pd, _CMP_NEQ_OQ, self, rhs)
385 }
386 #[inline(always)]
387 unsafe fn cmp_f32_eq(self, rhs: Self) -> Self {
388 self_from_const_op!(_mm256_cmp_ps, _CMP_EQ_OQ, self, rhs)
389 }
390 #[inline(always)]
391 unsafe fn cmp_f32_lt(self, rhs: Self) -> Self {
392 self_from_const_op!(_mm256_cmp_ps, _CMP_LT_OQ, self, rhs)
393 }
394 #[inline(always)]
395 unsafe fn cmp_f32_le(self, rhs: Self) -> Self {
396 self_from_const_op!(_mm256_cmp_ps, _CMP_LE_OQ, self, rhs)
397 }
398 #[inline(always)]
399 unsafe fn cmp_f32_gt(self, rhs: Self) -> Self {
400 self_from_const_op!(_mm256_cmp_ps, _CMP_GT_OQ, self, rhs)
401 }
402 #[inline(always)]
403 unsafe fn cmp_f32_ge(self, rhs: Self) -> Self {
404 self_from_const_op!(_mm256_cmp_ps, _CMP_GE_OQ, self, rhs)
405 }
406 #[inline(always)]
407 unsafe fn cmp_f32_neq(self, rhs: Self) -> Self {
408 self_from_const_op!(_mm256_cmp_ps, _CMP_NEQ_OQ, self, rhs)
409 }
410 #[inline(always)]
411 unsafe fn cmp_i64_eq(self, rhs: Self) -> Self {
412 self_from_op!(_mm256_cmpeq_epi64, self, rhs)
413 }
414 #[inline(always)]
415 unsafe fn cmp_i64_gt(self, rhs: Self) -> Self {
416 self_from_op!(_mm256_cmpgt_epi64, self, rhs)
417 }
418 #[inline(always)]
419 unsafe fn cmp_i32_eq(self, rhs: Self) -> Self {
420 self_from_op!(_mm256_cmpeq_epi32, self, rhs)
421 }
422 #[inline(always)]
423 unsafe fn cmp_i32_gt(self, rhs: Self) -> Self {
424 self_from_op!(_mm256_cmpgt_epi32, self, rhs)
425 }
426 #[inline(always)]
427 unsafe fn cmp_i16_eq(self, rhs: Self) -> Self {
428 self_from_op!(_mm256_cmpeq_epi16, self, rhs)
429 }
430 #[inline(always)]
431 unsafe fn cmp_i16_gt(self, rhs: Self) -> Self {
432 self_from_op!(_mm256_cmpgt_epi16, self, rhs)
433 }
434 #[inline(always)]
435 unsafe fn cmp_i8_eq(self, rhs: Self) -> Self {
436 self_from_op!(_mm256_cmpeq_epi8, self, rhs)
437 }
438 #[inline(always)]
439 unsafe fn cmp_i8_gt(self, rhs: Self) -> Self {
440 self_from_op!(_mm256_cmpgt_epi8, self, rhs)
441 }
442
443 #[inline(always)]
444 unsafe fn max_f64(self, rhs: Self) -> Self {
445 self_from_op!(_mm256_max_pd, self, rhs)
446 }
447 #[inline(always)]
448 unsafe fn min_f64(self, rhs: Self) -> Self {
449 self_from_op!(_mm256_min_pd, self, rhs)
450 }
451 #[inline(always)]
452 unsafe fn max_f32(self, rhs: Self) -> Self {
453 self_from_op!(_mm256_max_ps, self, rhs)
454 }
455 #[inline(always)]
456 unsafe fn min_f32(self, rhs: Self) -> Self {
457 self_from_op!(_mm256_min_ps, self, rhs)
458 }
459 #[inline(always)]
460 unsafe fn max_i32(self, rhs: Self) -> Self {
461 self_from_op!(_mm256_max_epi32, self, rhs)
462 }
463 #[inline(always)]
464 unsafe fn min_i32(self, rhs: Self) -> Self {
465 self_from_op!(_mm256_min_epi32, self, rhs)
466 }
467 #[inline(always)]
468 unsafe fn max_i16(self, rhs: Self) -> Self {
469 self_from_op!(_mm256_max_epi16, self, rhs)
470 }
471 #[inline(always)]
472 unsafe fn min_i16(self, rhs: Self) -> Self {
473 self_from_op!(_mm256_min_epi16, self, rhs)
474 }
475 #[inline(always)]
476 unsafe fn max_i8(self, rhs: Self) -> Self {
477 self_from_op!(_mm256_max_epi8, self, rhs)
478 }
479 #[inline(always)]
480 unsafe fn min_i8(self, rhs: Self) -> Self {
481 self_from_op!(_mm256_min_epi8, self, rhs)
482 }
483 #[inline(always)]
484 unsafe fn max_u32(self, rhs: Self) -> Self {
485 self_from_op!(_mm256_max_epu32, self, rhs)
486 }
487 #[inline(always)]
488 unsafe fn min_u32(self, rhs: Self) -> Self {
489 self_from_op!(_mm256_min_epu32, self, rhs)
490 }
491 #[inline(always)]
492 unsafe fn max_u16(self, rhs: Self) -> Self {
493 self_from_op!(_mm256_max_epu16, self, rhs)
494 }
495 #[inline(always)]
496 unsafe fn min_u16(self, rhs: Self) -> Self {
497 self_from_op!(_mm256_min_epu16, self, rhs)
498 }
499 #[inline(always)]
500 unsafe fn max_u8(self, rhs: Self) -> Self {
501 self_from_op!(_mm256_max_epu8, self, rhs)
502 }
503 #[inline(always)]
504 unsafe fn min_u8(self, rhs: Self) -> Self {
505 self_from_op!(_mm256_min_epu8, self, rhs)
506 }
507}
508
509impl SimdSplatImpl for Avx2Reg {
511 #[inline(always)]
512 unsafe fn splat_64<T>(val: T) -> Self {
513 self_from_op!(_mm256_set1_epi64x, val)
514 }
515 #[inline(always)]
516 unsafe fn splat_32<T>(val: T) -> Self {
517 self_from_op!(_mm256_set1_epi32, val)
518 }
519 #[inline(always)]
520 unsafe fn splat_16<T>(val: T) -> Self {
521 self_from_op!(_mm256_set1_epi16, val)
522 }
523 #[inline(always)]
524 unsafe fn splat_8<T>(val: T) -> Self {
525 self_from_op!(_mm256_set1_epi8, val)
526 }
527}
528
529impl SimdGatherImpl for Avx2Reg {
530 #[inline(always)]
531 unsafe fn gather_32_from_32<T, const B: i32>(self, ptr: *const T) -> Self {
532 self_from_const_op!(_mm256_i32gather_epi32, B, ptr, self)
533 }
534 #[inline(always)]
537 unsafe fn gather_64_from_64<T, const B: i32>(self, ptr: *const T) -> Self {
538 self_from_const_op!(_mm256_i64gather_epi64, B, ptr, self)
539 }
540}
541
542impl SimdSqrtImpl for Avx2Reg {
543 #[inline(always)]
544 unsafe fn sqrt_f64(self) -> Self {
545 self_from_op!(_mm256_sqrt_pd, self)
546 }
547 #[inline(always)]
548 unsafe fn sqrt_f32(self) -> Self {
549 self_from_op!(_mm256_sqrt_ps, self)
550 }
551 #[inline(always)]
552 unsafe fn rsqrt_f32(self) -> Self {
553 self_from_op!(_mm256_rsqrt_ps, self)
554 }
555}
556
557impl SimdAllBitsImpl for Avx2Reg {
558 #[inline(always)]
559 unsafe fn all_zero(self) -> bool {
560 execute_intrinsic!(_mm256_testz_si256, self, self) != 0
561 }
562}
563
564impl SimdNegateImpl for Avx2Reg {
565 #[inline(always)]
566 unsafe fn negate_f64(self) -> Self {
567 unsafe { Self::splat_64(-0.0f64).xor(self) }
568 }
569 #[inline(always)]
570 unsafe fn negate_f32(self) -> Self {
571 unsafe { Self::splat_32(-0.0f64).xor(self) }
572 }
573}
574
575impl SimdBlockShiftImpl for Avx2Reg {
576 #[inline(always)]
577 unsafe fn block_left_byte_shift<const N: i32>(self) -> Self {
578 self_from_const_op!(_mm256_bslli_epi128, N, self)
579 }
580 #[inline(always)]
581 unsafe fn block_right_byte_shift<const N: i32>(self) -> Self {
582 self_from_const_op!(_mm256_bsrli_epi128, N, self)
583 }
584}
585
586impl SimdMaskBitConversion for Avx2Reg {
587 #[inline(always)]
588 unsafe fn to_bits_64(self) -> u64 {
589 execute_intrinsic!(_mm256_movemask_pd, self) as u64
590 }
591 #[inline(always)]
592 unsafe fn to_bits_32(self) -> u64 {
593 execute_intrinsic!(_mm256_movemask_ps, self) as u64
594 }
595 #[inline(always)]
596 unsafe fn to_bits_8(self) -> u64 {
597 execute_intrinsic!(_mm256_movemask_epi8, self) as u64
598 }
599 #[inline(always)]
600 unsafe fn from_bits_64(bitmask: u64) -> Self {
601 let mask = self_from_op!(_mm256_set_epi64x, 1, 2, 4, 8);
602 let bits = self_from_op!(_mm256_set1_epi64x, bitmask as i64);
603 self_from_op!(_mm256_cmpgt_epi64, bits.and(mask), Self::zero())
604 }
605 #[inline(always)]
606 unsafe fn from_bits_32(bitmask: u64) -> Self {
607 let mask = self_from_op!(_mm256_set_epi32, 1, 2, 4, 8, 16, 32, 64, 128);
608 let bits = self_from_op!(_mm256_set1_epi32, bitmask as u32);
609 self_from_op!(_mm256_cmpgt_epi32, bits.and(mask), Self::zero())
610 }
611 #[inline(always)]
612 unsafe fn from_bits_16(bitmask: u64) -> Self {
613 #[rustfmt::skip]
614 let mask = self_from_op!(_mm256_set_epi16, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, -32768);
615 let bits = self_from_op!(_mm256_set1_epi16, bitmask as i16);
616 unsafe { self_from_op!(_mm256_cmpeq_epi16, bits.and(mask), Self::zero()).not() }
617 }
618 #[inline(always)]
619 unsafe fn from_bits_8(bitmask: u64) -> Self {
620 let b1 = bitmask as i8;
621 let b2 = (bitmask >> 8) as i8;
622 let b3 = (bitmask >> 16) as i8;
623 let b4 = (bitmask >> 24) as i8;
624 #[rustfmt::skip]
625 let mask = self_from_op!(_mm256_set_epi8,
626 1, 2, 4, 8, 16, 32, 64, -128,
627 1, 2, 4, 8, 16, 32, 64, -128,
628 1, 2, 4, 8, 16, 32, 64, -128,
629 1, 2, 4, 8, 16, 32, 64, -128
630 );
631 #[rustfmt::skip]
632 let bits = self_from_op!(_mm256_set_epi8,
633 b1, b1, b1, b1, b1, b1, b1, b1,
634 b2, b2, b2, b2, b2, b2, b2, b2,
635 b3, b3, b3, b3, b3, b3, b3, b3,
636 b4, b4, b4, b4, b4, b4, b4, b4
637 );
638 unsafe { self_from_op!(_mm256_cmpeq_epi8, bits.and(mask), Self::zero()).not() }
639 }
640}
641
642impl SimdLaneShiftImpl for Avx2Reg {
643 #[inline(always)]
644 unsafe fn left_lane_shift_32(self, n: u32) -> Self {
645 unsafe {
646 match n {
647 0 => self,
648 #[rustfmt::skip]
649 1 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 1, 2, 3, 4, 5, 6, 7, 0)),
650 #[rustfmt::skip]
651 2 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 2, 3, 4, 5, 6, 7, 0, 0)),
652 #[rustfmt::skip]
653 3 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 3, 4, 5, 6, 7, 0, 0, 0)),
654 #[rustfmt::skip]
655 4 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 4, 5, 6, 7, 0, 0, 0, 0)),
656 #[rustfmt::skip]
657 5 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 5, 6, 7, 0, 0, 0, 0, 0)),
658 #[rustfmt::skip]
659 6 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 6, 7, 0, 0, 0, 0, 0, 0)),
660 #[rustfmt::skip]
661 7 => Self::zero().blend_32::<1>(self).permute_32(self_from_op!(_mm256_setr_epi32, 7, 0, 0, 0, 0, 0, 0, 0)),
662 _ => Self::zero(),
663 }
664 }
665 }
666 #[inline(always)]
667 unsafe fn right_lane_shift_32(self, n: u32) -> Self {
668 unsafe {
669 match n {
670 0 => self,
671 1 => Self::zero()
672 .blend_32::<0x80>(self)
673 .permute_32(self_from_op!(_mm256_setr_epi32, 7, 0, 1, 2, 3, 4, 5, 6)),
674 2 => Self::zero()
675 .blend_32::<0x80>(self)
676 .permute_32(self_from_op!(_mm256_setr_epi32, 7, 7, 0, 1, 2, 3, 4, 5)),
677 3 => Self::zero()
678 .blend_32::<0x80>(self)
679 .permute_32(self_from_op!(_mm256_setr_epi32, 7, 7, 7, 0, 1, 2, 3, 4)),
680 4 => Self::zero()
681 .blend_32::<0x80>(self)
682 .permute_32(self_from_op!(_mm256_setr_epi32, 7, 7, 7, 7, 0, 1, 2, 3)),
683 5 => Self::zero()
684 .blend_32::<0x80>(self)
685 .permute_32(self_from_op!(_mm256_setr_epi32, 7, 7, 7, 7, 7, 0, 1, 2)),
686 6 => Self::zero()
687 .blend_32::<0x80>(self)
688 .permute_32(self_from_op!(_mm256_setr_epi32, 7, 7, 7, 7, 7, 7, 0, 1)),
689 7 => Self::zero()
690 .blend_32::<0x80>(self)
691 .permute_32(self_from_op!(_mm256_setr_epi32, 7, 7, 7, 7, 7, 7, 7, 0)),
692 _ => Self::zero(),
693 }
694 }
695 }
696}