Skip to main content

simply_simd/architectures/intrinsics/
sse.rs

1use std::arch::x86_64::*;
2use std::mem::{transmute, transmute_copy};
3
4use crate::architectures::interface::*;
5use crate::architectures::macros::*;
6
7#[derive(Copy, Clone)]
8#[repr(transparent)]
9pub struct SseReg(pub __m128i);
10impl SimdArch for SseReg {}
11impl MaskArch for SseReg {}
12
13impl SimdAddImpl for SseReg {
14    #[inline(always)]
15    unsafe fn f64_add(self, rhs: Self) -> Self {
16        self_from_op!(_mm_add_pd, self, rhs)
17    }
18    #[inline(always)]
19    unsafe fn f32_add(self, rhs: Self) -> Self {
20        self_from_op!(_mm_add_ps, self, rhs)
21    }
22    #[inline(always)]
23    unsafe fn i64_add(self, rhs: Self) -> Self {
24        self_from_op!(_mm_add_epi64, self, rhs)
25    }
26    #[inline(always)]
27    unsafe fn i32_add(self, rhs: Self) -> Self {
28        self_from_op!(_mm_add_epi32, self, rhs)
29    }
30    #[inline(always)]
31    unsafe fn i16_add(self, rhs: Self) -> Self {
32        self_from_op!(_mm_add_epi16, self, rhs)
33    }
34    #[inline(always)]
35    unsafe fn i8_add(self, rhs: Self) -> Self {
36        self_from_op!(_mm_add_epi8, self, rhs)
37    }
38}
39
40impl SimdSubImpl for SseReg {
41    #[inline(always)]
42    unsafe fn f64_sub(self, rhs: Self) -> Self {
43        self_from_op!(_mm_sub_pd, self, rhs)
44    }
45    #[inline(always)]
46    unsafe fn f32_sub(self, rhs: Self) -> Self {
47        self_from_op!(_mm_sub_ps, self, rhs)
48    }
49    #[inline(always)]
50    unsafe fn i64_sub(self, rhs: Self) -> Self {
51        self_from_op!(_mm_sub_epi64, self, rhs)
52    }
53    #[inline(always)]
54    unsafe fn i32_sub(self, rhs: Self) -> Self {
55        self_from_op!(_mm_sub_epi32, self, rhs)
56    }
57    #[inline(always)]
58    unsafe fn i16_sub(self, rhs: Self) -> Self {
59        self_from_op!(_mm_sub_epi16, self, rhs)
60    }
61    #[inline(always)]
62    unsafe fn i8_sub(self, rhs: Self) -> Self {
63        self_from_op!(_mm_sub_epi8, self, rhs)
64    }
65}
66
67impl SimdMulImpl for SseReg {
68    #[inline(always)]
69    unsafe fn f64_mul(self, rhs: Self) -> Self {
70        self_from_op!(_mm_mul_pd, self, rhs)
71    }
72    #[inline(always)]
73    unsafe fn f32_mul(self, rhs: Self) -> Self {
74        self_from_op!(_mm_mul_ps, self, rhs)
75    }
76    #[inline(always)]
77    unsafe fn i32_mul(self, rhs: Self) -> Self {
78        self_from_op!(_mm_mullo_epi32, self, rhs)
79    }
80    #[inline(always)]
81    unsafe fn i16_mul(self, rhs: Self) -> Self {
82        self_from_op!(_mm_mullo_epi16, self, rhs)
83    }
84}
85
86impl SimdDivImpl for SseReg {
87    #[inline(always)]
88    unsafe fn f64_div(self, rhs: Self) -> Self {
89        self_from_op!(_mm_div_pd, self, rhs)
90    }
91    #[inline(always)]
92    unsafe fn f32_div(self, rhs: Self) -> Self {
93        self_from_op!(_mm_div_ps, self, rhs)
94    }
95}
96
97impl SimdBitwiseImpl for SseReg {
98    #[inline(always)]
99    unsafe fn and(self, rhs: Self) -> Self {
100        self_from_op!(_mm_and_si128, self, rhs)
101    }
102    #[inline(always)]
103    unsafe fn or(self, rhs: Self) -> Self {
104        self_from_op!(_mm_or_si128, self, rhs)
105    }
106    #[inline(always)]
107    unsafe fn xor(self, rhs: Self) -> Self {
108        self_from_op!(_mm_xor_si128, self, rhs)
109    }
110    #[inline(always)]
111    unsafe fn not(self) -> Self {
112        unsafe { Self(self.xor(Self::splat_32(!0)).0) }
113    }
114    #[inline(always)]
115    unsafe fn and_not(self, rhs: Self) -> Self {
116        self_from_op!(_mm_andnot_si128, rhs, self)
117    }
118}
119
120macro_rules! scalar_shift {
121    ($self:expr, $rhs:expr, $lanes:expr, $elem:ty, $uelem:ty, $op:tt) => {{
122        let mut a = [<$elem>::default(); $lanes];
123        let mut b = [<$elem>::default(); $lanes];
124        _mm_storeu_si128(a.as_mut_ptr() as *mut __m128i, $self.0);
125        _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, $rhs.0);
126        let width = (std::mem::size_of::<$elem>() * 8) as u32;
127        let mut out = [<$elem>::default(); $lanes];
128        for i in 0..$lanes {
129            let shift = b[i] as $uelem as u32;
130            out[i] = scalar_shift!(@apply $op, a[i], shift, width, $elem, $uelem);
131        }
132        Self(_mm_loadu_si128(out.as_ptr() as *const __m128i))
133    }};
134    (@apply sll, $val:expr, $shift:expr, $width:expr, $elem:ty, $uelem:ty) => {
135        if $shift >= $width { 0 } else { (($val as $uelem) << $shift) as $elem }
136    };
137    (@apply srl, $val:expr, $shift:expr, $width:expr, $elem:ty, $uelem:ty) => {
138        if $shift >= $width { 0 } else { (($val as $uelem) >> $shift) as $elem }
139    };
140    (@apply sra, $val:expr, $shift:expr, $width:expr, $elem:ty, $uelem:ty) => {
141        $val >> $shift.min($width - 1)
142    };
143}
144
145impl SimdShiftImpl for SseReg {
146    #[inline(always)]
147    unsafe fn sllv_64(self, rhs: Self) -> Self {
148        unsafe { scalar_shift!(self, rhs, 2, i64, u64, sll) }
149    }
150    #[inline(always)]
151    unsafe fn srlv_64(self, rhs: Self) -> Self {
152        unsafe { scalar_shift!(self, rhs, 2, i64, u64, srl) }
153    }
154    #[inline(always)]
155    unsafe fn srav_64(self, rhs: Self) -> Self {
156        unsafe { scalar_shift!(self, rhs, 2, i64, u64, sra) }
157    }
158    #[inline(always)]
159    unsafe fn sllv_32(self, rhs: Self) -> Self {
160        unsafe { scalar_shift!(self, rhs, 4, i32, u32, sll) }
161    }
162    #[inline(always)]
163    unsafe fn srlv_32(self, rhs: Self) -> Self {
164        unsafe { scalar_shift!(self, rhs, 4, i32, u32, srl) }
165    }
166    #[inline(always)]
167    unsafe fn srav_32(self, rhs: Self) -> Self {
168        unsafe { scalar_shift!(self, rhs, 4, i32, u32, sra) }
169    }
170    #[inline(always)]
171    unsafe fn sllv_16(self, rhs: Self) -> Self {
172        unsafe { scalar_shift!(self, rhs, 8, i16, u16, sll) }
173    }
174    #[inline(always)]
175    unsafe fn srlv_16(self, rhs: Self) -> Self {
176        unsafe { scalar_shift!(self, rhs, 8, i16, u16, srl) }
177    }
178    #[inline(always)]
179    unsafe fn srav_16(self, rhs: Self) -> Self {
180        unsafe { scalar_shift!(self, rhs, 8, i16, u16, sra) }
181    }
182}
183impl SimdLoadImpl for SseReg {
184    type MaskType = Self;
185
186    #[inline(always)]
187    unsafe fn load_aligned<T>(ptr: *const T) -> Self {
188        self_from_op!(_mm_load_si128, ptr)
189    }
190    #[inline(always)]
191    unsafe fn load_unaligned<T>(ptr: *const T) -> Self {
192        self_from_op!(_mm_loadu_si128, ptr)
193    }
194
195    #[inline(always)]
196    unsafe fn masked_load_64<T>(ptr: *const T, mask: Self::MaskType) -> Self {
197        unsafe {
198            let mut m = [0i64; 2];
199            _mm_storeu_si128(m.as_mut_ptr() as *mut __m128i, mask.0);
200            let src = ptr as *const i64;
201            let mut out = [0i64; 2];
202            for i in 0..2 {
203                // AVX2 semantics: only the top bit of each lane's mask element is checked
204                if (m[i] as u64) & (1 << 63) != 0 {
205                    out[i] = *src.add(i);
206                }
207            }
208            Self(_mm_loadu_si128(out.as_ptr() as *const __m128i))
209        }
210    }
211
212    #[inline(always)]
213    unsafe fn masked_load_32<T>(ptr: *const T, mask: Self::MaskType) -> Self {
214        unsafe {
215            let mut m = [0i32; 4];
216            _mm_storeu_si128(m.as_mut_ptr() as *mut __m128i, mask.0);
217            let src = ptr as *const i32;
218            let mut out = [0i32; 4];
219            for i in 0..4 {
220                if (m[i] as u32) & (1 << 31) != 0 {
221                    out[i] = *src.add(i);
222                }
223            }
224            Self(_mm_loadu_si128(out.as_ptr() as *const __m128i))
225        }
226    }
227}
228
229impl SimdStoreImpl for SseReg {
230    type MaskType = Self;
231
232    #[inline(always)]
233    unsafe fn store_aligned<T>(self, ptr: *mut T) {
234        execute_intrinsic!(_mm_store_si128, ptr, self);
235    }
236    #[inline(always)]
237    unsafe fn store_unaligned<T>(self, ptr: *mut T) {
238        execute_intrinsic!(_mm_storeu_si128, ptr, self);
239    }
240
241    #[inline(always)]
242    unsafe fn masked_store_64<T>(self, ptr: *mut T, mask: Self::MaskType) {
243        unsafe {
244            let mut m = [0i64; 2];
245            let mut v = [0i64; 2];
246            _mm_storeu_si128(m.as_mut_ptr() as *mut __m128i, mask.0);
247            _mm_storeu_si128(v.as_mut_ptr() as *mut __m128i, self.0);
248            let dst = ptr as *mut i64;
249            for i in 0..2 {
250                if (m[i] as u64) & (1 << 63) != 0 {
251                    *dst.add(i) = v[i];
252                }
253            }
254        }
255    }
256
257    #[inline(always)]
258    unsafe fn masked_store_32<T>(self, ptr: *mut T, mask: Self::MaskType) {
259        unsafe {
260            let mut m = [0i32; 4];
261            let mut v = [0i32; 4];
262            _mm_storeu_si128(m.as_mut_ptr() as *mut __m128i, mask.0);
263            _mm_storeu_si128(v.as_mut_ptr() as *mut __m128i, self.0);
264            let dst = ptr as *mut i32;
265            for i in 0..4 {
266                if (m[i] as u32) & (1 << 31) != 0 {
267                    *dst.add(i) = v[i];
268                }
269            }
270        }
271    }
272}
273
274impl SimdZeroImpl for SseReg {
275    #[inline(always)]
276    unsafe fn zero() -> Self {
277        self_from_op!(_mm_setzero_si128,)
278    }
279}
280
281impl SimdFloatCastsImpl for SseReg {
282    #[inline(always)]
283    unsafe fn float_to_int_trunc(self) -> Self {
284        self_from_op!(_mm_cvttps_epi32, self)
285    }
286    #[inline(always)]
287    unsafe fn float_to_int_round(self) -> Self {
288        self_from_op!(_mm_cvtps_epi32, self)
289    }
290}
291
292impl SimdIntCastsImpl for SseReg {
293    #[inline(always)]
294    unsafe fn int_to_float(self) -> Self {
295        self_from_op!(_mm_cvtepi32_ps, self)
296    }
297}
298
299impl SimdPermuteImpl for SseReg {
300    #[inline(always)]
301    unsafe fn permute_32(self, rhs: Self) -> Self {
302        unsafe {
303            let mut a = [0.0f32; 4];
304            let mut idx = [0i32; 4];
305            _mm_storeu_ps(a.as_mut_ptr(), std::mem::transmute(self.0));
306            _mm_storeu_si128(idx.as_mut_ptr() as *mut __m128i, rhs.0);
307            let mut out = [0.0f32; 4];
308            for i in 0..4 {
309                out[i] = a[(idx[i] & 0b11) as usize]; // low 2 bits select lane, same as hardware
310            }
311            Self(std::mem::transmute(_mm_loadu_ps(out.as_ptr())))
312        }
313    }
314
315    #[inline(always)]
316    unsafe fn permute_8(self, rhs: Self) -> Self {
317        self_from_op!(_mm_shuffle_epi8, self, rhs)
318    }
319}
320impl SimdVariableBlendImpl for SseReg {
321    type VecType = Self;
322    #[inline(always)]
323    unsafe fn vblend_64(self, true_values: Self::VecType, false_values: Self::VecType) -> Self {
324        self_from_op!(_mm_blendv_pd, false_values, true_values, self)
325    }
326    #[inline(always)]
327    unsafe fn vblend_32(self, true_values: Self::VecType, false_values: Self::VecType) -> Self {
328        self_from_op!(_mm_blendv_ps, false_values, true_values, self)
329    }
330    #[inline(always)]
331    unsafe fn vblend_8(self, true_values: Self::VecType, false_values: Self::VecType) -> Self {
332        self_from_op!(_mm_blendv_epi8, false_values, true_values, self)
333    }
334}
335
336impl SimdImmediateBlendImpl for SseReg {
337    #[inline(always)]
338    unsafe fn blend_64<const N: i32>(self, false_values: Self) -> Self {
339        self_from_const_op!(_mm_blend_pd, N, false_values, self)
340    }
341    #[inline(always)]
342    unsafe fn blend_32<const N: i32>(self, false_values: Self) -> Self {
343        self_from_const_op!(_mm_blend_ps, N, false_values, self)
344    }
345}
346
347impl SimdMulAddImpl for SseReg {
348    #[inline(always)]
349    unsafe fn mul_add_f64(self, mult: Self, add: Self) -> Self {
350        self_from_op!(_mm_add_pd, self_from_op!(_mm_mul_pd, self, mult), add)
351    }
352    #[inline(always)]
353    unsafe fn mul_sub_f64(self, mult: Self, sub: Self) -> Self {
354        self_from_op!(_mm_sub_pd, self_from_op!(_mm_mul_pd, self, mult), sub)
355    }
356    #[inline(always)]
357    unsafe fn negated_mul_add_f64(self, mult: Self, add: Self) -> Self {
358        self_from_op!(_mm_sub_pd, add, self_from_op!(_mm_mul_pd, self, mult))
359    }
360    #[inline(always)]
361    unsafe fn negated_mul_sub_f64(self, mult: Self, sub: Self) -> Self {
362        let neg = self_from_op!(_mm_mul_pd, self, mult);
363        self_from_op!(
364            _mm_sub_pd,
365            self_from_op!(_mm_xor_pd, neg, Self::splat_64(-0.0f64).0),
366            sub
367        )
368    }
369
370    #[inline(always)]
371    unsafe fn mul_add_f32(self, mult: Self, add: Self) -> Self {
372        self_from_op!(_mm_add_ps, self_from_op!(_mm_mul_ps, self, mult), add)
373    }
374    #[inline(always)]
375    unsafe fn mul_sub_f32(self, mult: Self, sub: Self) -> Self {
376        self_from_op!(_mm_sub_ps, self_from_op!(_mm_mul_ps, self, mult), sub)
377    }
378    #[inline(always)]
379    unsafe fn negated_mul_add_f32(self, mult: Self, add: Self) -> Self {
380        self_from_op!(_mm_sub_ps, add, self_from_op!(_mm_mul_ps, self, mult))
381    }
382    #[inline(always)]
383    unsafe fn negated_mul_sub_f32(self, mult: Self, sub: Self) -> Self {
384        let neg = self_from_op!(_mm_mul_ps, self, mult);
385        self_from_op!(
386            _mm_sub_ps,
387            self_from_op!(_mm_xor_ps, neg, Self::splat_32(-0.0f32).0),
388            sub
389        )
390    }
391}
392
393impl SimdRoundImpl for SseReg {
394    #[inline(always)]
395    unsafe fn round_f64(self) -> Self {
396        self_from_const_op!(
397            _mm_round_pd,
398            _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC,
399            self
400        )
401    }
402    #[inline(always)]
403    unsafe fn round_f32(self) -> Self {
404        self_from_const_op!(
405            _mm_round_ps,
406            _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC,
407            self
408        )
409    }
410    #[inline(always)]
411    unsafe fn floor_f64(self) -> Self {
412        self_from_const_op!(
413            _mm_round_pd,
414            _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC,
415            self
416        )
417    }
418    #[inline(always)]
419    unsafe fn floor_f32(self) -> Self {
420        self_from_const_op!(
421            _mm_round_ps,
422            _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC,
423            self
424        )
425    }
426    #[inline(always)]
427    unsafe fn ceil_f64(self) -> Self {
428        self_from_const_op!(
429            _mm_round_pd,
430            _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC,
431            self
432        )
433    }
434    #[inline(always)]
435    unsafe fn ceil_f32(self) -> Self {
436        self_from_const_op!(
437            _mm_round_ps,
438            _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC,
439            self
440        )
441    }
442}
443
444impl SimdPartialOrdImpl for SseReg {
445    type MaskType = Self;
446    #[inline(always)]
447    unsafe fn cmp_f64_eq(self, rhs: Self) -> Self {
448        self_from_op!(_mm_cmpeq_pd, self, rhs)
449    }
450    #[inline(always)]
451    unsafe fn cmp_f64_lt(self, rhs: Self) -> Self {
452        self_from_op!(_mm_cmplt_pd, self, rhs)
453    }
454    #[inline(always)]
455    unsafe fn cmp_f64_le(self, rhs: Self) -> Self {
456        self_from_op!(_mm_cmple_pd, self, rhs)
457    }
458    #[inline(always)]
459    unsafe fn cmp_f64_gt(self, rhs: Self) -> Self {
460        self_from_op!(_mm_cmpgt_pd, self, rhs)
461    }
462    #[inline(always)]
463    unsafe fn cmp_f64_ge(self, rhs: Self) -> Self {
464        self_from_op!(_mm_cmpge_pd, self, rhs)
465    }
466    #[inline(always)]
467    unsafe fn cmp_f64_neq(self, rhs: Self) -> Self {
468        self_from_op!(_mm_cmpneq_pd, self, rhs)
469    }
470    #[inline(always)]
471    unsafe fn cmp_f32_eq(self, rhs: Self) -> Self {
472        self_from_op!(_mm_cmpeq_ps, self, rhs)
473    }
474    #[inline(always)]
475    unsafe fn cmp_f32_lt(self, rhs: Self) -> Self {
476        self_from_op!(_mm_cmplt_ps, self, rhs)
477    }
478    #[inline(always)]
479    unsafe fn cmp_f32_le(self, rhs: Self) -> Self {
480        self_from_op!(_mm_cmple_ps, self, rhs)
481    }
482    #[inline(always)]
483    unsafe fn cmp_f32_gt(self, rhs: Self) -> Self {
484        self_from_op!(_mm_cmpgt_ps, self, rhs)
485    }
486    #[inline(always)]
487    unsafe fn cmp_f32_ge(self, rhs: Self) -> Self {
488        self_from_op!(_mm_cmpge_ps, self, rhs)
489    }
490    #[inline(always)]
491    unsafe fn cmp_f32_neq(self, rhs: Self) -> Self {
492        self_from_op!(_mm_cmpneq_ps, self, rhs)
493    }
494    #[inline(always)]
495    unsafe fn cmp_i64_eq(self, rhs: Self) -> Self {
496        self_from_op!(_mm_cmpeq_epi64, self, rhs)
497    }
498    #[inline(always)]
499    unsafe fn cmp_i64_gt(self, rhs: Self) -> Self {
500        self_from_op!(_mm_cmpgt_epi64, self, rhs)
501    }
502    #[inline(always)]
503    unsafe fn cmp_i32_eq(self, rhs: Self) -> Self {
504        self_from_op!(_mm_cmpeq_epi32, self, rhs)
505    }
506    #[inline(always)]
507    unsafe fn cmp_i32_gt(self, rhs: Self) -> Self {
508        self_from_op!(_mm_cmpgt_epi32, self, rhs)
509    }
510    #[inline(always)]
511    unsafe fn cmp_i16_eq(self, rhs: Self) -> Self {
512        self_from_op!(_mm_cmpeq_epi16, self, rhs)
513    }
514    #[inline(always)]
515    unsafe fn cmp_i16_gt(self, rhs: Self) -> Self {
516        self_from_op!(_mm_cmpgt_epi16, self, rhs)
517    }
518    #[inline(always)]
519    unsafe fn cmp_i8_eq(self, rhs: Self) -> Self {
520        self_from_op!(_mm_cmpeq_epi8, self, rhs)
521    }
522    #[inline(always)]
523    unsafe fn cmp_i8_gt(self, rhs: Self) -> Self {
524        self_from_op!(_mm_cmpgt_epi8, self, rhs)
525    }
526
527    #[inline(always)]
528    unsafe fn max_f64(self, rhs: Self) -> Self {
529        self_from_op!(_mm_max_pd, self, rhs)
530    }
531    #[inline(always)]
532    unsafe fn min_f64(self, rhs: Self) -> Self {
533        self_from_op!(_mm_min_pd, self, rhs)
534    }
535    #[inline(always)]
536    unsafe fn max_f32(self, rhs: Self) -> Self {
537        self_from_op!(_mm_max_ps, self, rhs)
538    }
539    #[inline(always)]
540    unsafe fn min_f32(self, rhs: Self) -> Self {
541        self_from_op!(_mm_min_ps, self, rhs)
542    }
543    #[inline(always)]
544    unsafe fn max_i32(self, rhs: Self) -> Self {
545        self_from_op!(_mm_max_epi32, self, rhs)
546    }
547    #[inline(always)]
548    unsafe fn min_i32(self, rhs: Self) -> Self {
549        self_from_op!(_mm_min_epi32, self, rhs)
550    }
551    #[inline(always)]
552    unsafe fn max_i16(self, rhs: Self) -> Self {
553        self_from_op!(_mm_max_epi16, self, rhs)
554    }
555    #[inline(always)]
556    unsafe fn min_i16(self, rhs: Self) -> Self {
557        self_from_op!(_mm_min_epi16, self, rhs)
558    }
559    #[inline(always)]
560    unsafe fn max_i8(self, rhs: Self) -> Self {
561        self_from_op!(_mm_max_epi8, self, rhs)
562    }
563    #[inline(always)]
564    unsafe fn min_i8(self, rhs: Self) -> Self {
565        self_from_op!(_mm_min_epi8, self, rhs)
566    }
567    #[inline(always)]
568    unsafe fn max_u32(self, rhs: Self) -> Self {
569        self_from_op!(_mm_max_epu32, self, rhs)
570    }
571    #[inline(always)]
572    unsafe fn min_u32(self, rhs: Self) -> Self {
573        self_from_op!(_mm_min_epu32, self, rhs)
574    }
575    #[inline(always)]
576    unsafe fn max_u16(self, rhs: Self) -> Self {
577        self_from_op!(_mm_max_epu16, self, rhs)
578    }
579    #[inline(always)]
580    unsafe fn min_u16(self, rhs: Self) -> Self {
581        self_from_op!(_mm_min_epu16, self, rhs)
582    }
583    #[inline(always)]
584    unsafe fn max_u8(self, rhs: Self) -> Self {
585        self_from_op!(_mm_max_epu8, self, rhs)
586    }
587    #[inline(always)]
588    unsafe fn min_u8(self, rhs: Self) -> Self {
589        self_from_op!(_mm_min_epu8, self, rhs)
590    }
591}
592
593// TODO: Make a custom trait for handling this transmutation into i*.
594impl SimdSplatImpl for SseReg {
595    #[inline(always)]
596    unsafe fn splat_64<T>(val: T) -> Self {
597        self_from_op!(_mm_set1_epi64x, val)
598    }
599    #[inline(always)]
600    unsafe fn splat_32<T>(val: T) -> Self {
601        self_from_op!(_mm_set1_epi32, val)
602    }
603    #[inline(always)]
604    unsafe fn splat_16<T>(val: T) -> Self {
605        self_from_op!(_mm_set1_epi16, val)
606    }
607    #[inline(always)]
608    unsafe fn splat_8<T>(val: T) -> Self {
609        self_from_op!(_mm_set1_epi8, val)
610    }
611}
612
613impl SimdGatherImpl for SseReg {
614    #[inline(always)]
615    unsafe fn gather_32_from_32<T, const B: i32>(self, ptr: *const T) -> Self {
616        unsafe {
617            let mut idx = [0i32; 4];
618            _mm_storeu_si128(idx.as_mut_ptr() as *mut __m128i, self.0);
619            let base = ptr as *const u8;
620            let mut out = [0i32; 4];
621            for i in 0..4 {
622                let byte_offset = (idx[i] as isize) * (B as isize);
623                out[i] = *(base.offset(byte_offset) as *const i32);
624            }
625            Self(_mm_loadu_si128(out.as_ptr() as *const __m128i))
626        }
627    }
628
629    #[inline(always)]
630    unsafe fn gather_64_from_64<T, const B: i32>(self, ptr: *const T) -> Self {
631        unsafe {
632            let mut idx = [0i64; 2];
633            _mm_storeu_si128(idx.as_mut_ptr() as *mut __m128i, self.0);
634            let base = ptr as *const u8;
635            let mut out = [0i64; 2];
636            for i in 0..2 {
637                let byte_offset = (idx[i] as isize) * (B as isize);
638                out[i] = *(base.offset(byte_offset) as *const i64);
639            }
640            Self(_mm_loadu_si128(out.as_ptr() as *const __m128i))
641        }
642    }
643}
644
645impl SimdSqrtImpl for SseReg {
646    #[inline(always)]
647    unsafe fn sqrt_f64(self) -> Self {
648        self_from_op!(_mm_sqrt_pd, self)
649    }
650    #[inline(always)]
651    unsafe fn sqrt_f32(self) -> Self {
652        self_from_op!(_mm_sqrt_ps, self)
653    }
654    #[inline(always)]
655    unsafe fn rsqrt_f32(self) -> Self {
656        self_from_op!(_mm_rsqrt_ps, self)
657    }
658}
659
660impl SimdAllBitsImpl for SseReg {
661    #[inline(always)]
662    unsafe fn all_zero(self) -> bool {
663        execute_intrinsic!(_mm_testz_si128, self, self) != 0
664    }
665}
666
667impl SimdNegateImpl for SseReg {
668    #[inline(always)]
669    unsafe fn negate_f64(self) -> Self {
670        unsafe { Self::splat_64(-0.0f64).xor(self) }
671    }
672    #[inline(always)]
673    unsafe fn negate_f32(self) -> Self {
674        unsafe { Self::splat_32(-0.0f64).xor(self) }
675    }
676}
677
678// TODO: THIS IS BACKWARDS COMPARED TO NON-BLOCKED, figure out why and fix.
679impl SimdBlockShiftImpl for SseReg {
680    #[inline(always)]
681    unsafe fn block_left_byte_shift<const N: i32>(self) -> Self {
682        self_from_const_op!(_mm_bslli_si128, N, self)
683    }
684    #[inline(always)]
685    unsafe fn block_right_byte_shift<const N: i32>(self) -> Self {
686        self_from_const_op!(_mm_bsrli_si128, N, self)
687    }
688}
689
690impl SimdMaskBitConversion for SseReg {
691    #[inline(always)]
692    unsafe fn to_bits_64(self) -> u64 {
693        execute_intrinsic!(_mm_movemask_pd, self) as u64
694    }
695    #[inline(always)]
696    unsafe fn to_bits_32(self) -> u64 {
697        execute_intrinsic!(_mm_movemask_ps, self) as u64
698    }
699    #[inline(always)]
700    unsafe fn to_bits_8(self) -> u64 {
701        execute_intrinsic!(_mm_movemask_epi8, self) as u64
702    }
703    #[inline(always)]
704    unsafe fn from_bits_64(bitmask: u64) -> Self {
705        let mask = self_from_op!(_mm_set_epi64x, 1, 2);
706        let bits = self_from_op!(_mm_set1_epi64x, bitmask as i64);
707        self_from_op!(_mm_cmpgt_epi64, bits.and(mask), Self::zero()) //TODO: only sse4.2?
708    }
709    #[inline(always)]
710    unsafe fn from_bits_32(bitmask: u64) -> Self {
711        let mask = self_from_op!(_mm_set_epi32, 1, 2, 4, 8);
712        let bits = self_from_op!(_mm_set1_epi32, bitmask as u32);
713        self_from_op!(_mm_cmpgt_epi32, bits.and(mask), Self::zero())
714    }
715    #[inline(always)]
716    unsafe fn from_bits_16(bitmask: u64) -> Self {
717        #[rustfmt::skip]
718        let mask = self_from_op!(_mm_set_epi16, 1, 2, 4, 8, 16, 32, 64, 128);
719        let bits = self_from_op!(_mm_set1_epi16, bitmask as i16);
720        unsafe { self_from_op!(_mm_cmpeq_epi16, bits.and(mask), Self::zero()).not() }
721    }
722    #[inline(always)]
723    unsafe fn from_bits_8(bitmask: u64) -> Self {
724        let b1 = bitmask as i8;
725        let b2 = (bitmask >> 8) as i8;
726        #[rustfmt::skip]
727        let mask = self_from_op!(_mm_set_epi8,
728            1, 2, 4, 8, 16, 32, 64, -128,
729            1, 2, 4, 8, 16, 32, 64, -128
730        );
731        #[rustfmt::skip]
732        let bits = self_from_op!(_mm_set_epi8,
733            b1, b1, b1, b1, b1, b1, b1, b1,
734            b2, b2, b2, b2, b2, b2, b2, b2
735        );
736        unsafe { self_from_op!(_mm_cmpeq_epi8, bits.and(mask), Self::zero()).not() }
737    }
738}
739
740impl SimdLaneShiftImpl for SseReg {
741    #[inline(always)]
742    unsafe fn left_lane_shift_32(self, n: u32) -> Self {
743        match n {
744            0 => self,
745            1 => self_from_const_op!(_mm_bsrli_si128, 4, self),
746            2 => self_from_const_op!(_mm_bsrli_si128, 8, self),
747            3 => self_from_const_op!(_mm_bsrli_si128, 12, self),
748            _ => unsafe { Self::zero() },
749        }
750    }
751    #[inline(always)]
752    unsafe fn right_lane_shift_32(self, n: u32) -> Self {
753        match n {
754            0 => self,
755            1 => self_from_const_op!(_mm_bslli_si128, 4, self),
756            2 => self_from_const_op!(_mm_bslli_si128, 8, self),
757            3 => self_from_const_op!(_mm_bslli_si128, 12, self),
758            _ => unsafe { Self::zero() },
759        }
760    }
761}