1use std::arch::x86_64::*;
2use std::mem::{transmute, transmute_copy};
3
4use crate::architectures::interface::*;
5use crate::architectures::macros::*;
6
7#[derive(Copy, Clone)]
8#[repr(transparent)]
9pub struct SseReg(pub __m128i);
10impl SimdArch for SseReg {}
11impl MaskArch for SseReg {}
12
13impl SimdAddImpl for SseReg {
14 #[inline(always)]
15 unsafe fn f64_add(self, rhs: Self) -> Self {
16 self_from_op!(_mm_add_pd, self, rhs)
17 }
18 #[inline(always)]
19 unsafe fn f32_add(self, rhs: Self) -> Self {
20 self_from_op!(_mm_add_ps, self, rhs)
21 }
22 #[inline(always)]
23 unsafe fn i64_add(self, rhs: Self) -> Self {
24 self_from_op!(_mm_add_epi64, self, rhs)
25 }
26 #[inline(always)]
27 unsafe fn i32_add(self, rhs: Self) -> Self {
28 self_from_op!(_mm_add_epi32, self, rhs)
29 }
30 #[inline(always)]
31 unsafe fn i16_add(self, rhs: Self) -> Self {
32 self_from_op!(_mm_add_epi16, self, rhs)
33 }
34 #[inline(always)]
35 unsafe fn i8_add(self, rhs: Self) -> Self {
36 self_from_op!(_mm_add_epi8, self, rhs)
37 }
38}
39
40impl SimdSubImpl for SseReg {
41 #[inline(always)]
42 unsafe fn f64_sub(self, rhs: Self) -> Self {
43 self_from_op!(_mm_sub_pd, self, rhs)
44 }
45 #[inline(always)]
46 unsafe fn f32_sub(self, rhs: Self) -> Self {
47 self_from_op!(_mm_sub_ps, self, rhs)
48 }
49 #[inline(always)]
50 unsafe fn i64_sub(self, rhs: Self) -> Self {
51 self_from_op!(_mm_sub_epi64, self, rhs)
52 }
53 #[inline(always)]
54 unsafe fn i32_sub(self, rhs: Self) -> Self {
55 self_from_op!(_mm_sub_epi32, self, rhs)
56 }
57 #[inline(always)]
58 unsafe fn i16_sub(self, rhs: Self) -> Self {
59 self_from_op!(_mm_sub_epi16, self, rhs)
60 }
61 #[inline(always)]
62 unsafe fn i8_sub(self, rhs: Self) -> Self {
63 self_from_op!(_mm_sub_epi8, self, rhs)
64 }
65}
66
67impl SimdMulImpl for SseReg {
68 #[inline(always)]
69 unsafe fn f64_mul(self, rhs: Self) -> Self {
70 self_from_op!(_mm_mul_pd, self, rhs)
71 }
72 #[inline(always)]
73 unsafe fn f32_mul(self, rhs: Self) -> Self {
74 self_from_op!(_mm_mul_ps, self, rhs)
75 }
76 #[inline(always)]
77 unsafe fn i32_mul(self, rhs: Self) -> Self {
78 self_from_op!(_mm_mullo_epi32, self, rhs)
79 }
80 #[inline(always)]
81 unsafe fn i16_mul(self, rhs: Self) -> Self {
82 self_from_op!(_mm_mullo_epi16, self, rhs)
83 }
84}
85
86impl SimdDivImpl for SseReg {
87 #[inline(always)]
88 unsafe fn f64_div(self, rhs: Self) -> Self {
89 self_from_op!(_mm_div_pd, self, rhs)
90 }
91 #[inline(always)]
92 unsafe fn f32_div(self, rhs: Self) -> Self {
93 self_from_op!(_mm_div_ps, self, rhs)
94 }
95}
96
97impl SimdBitwiseImpl for SseReg {
98 #[inline(always)]
99 unsafe fn and(self, rhs: Self) -> Self {
100 self_from_op!(_mm_and_si128, self, rhs)
101 }
102 #[inline(always)]
103 unsafe fn or(self, rhs: Self) -> Self {
104 self_from_op!(_mm_or_si128, self, rhs)
105 }
106 #[inline(always)]
107 unsafe fn xor(self, rhs: Self) -> Self {
108 self_from_op!(_mm_xor_si128, self, rhs)
109 }
110 #[inline(always)]
111 unsafe fn not(self) -> Self {
112 unsafe { Self(self.xor(Self::splat_32(!0)).0) }
113 }
114 #[inline(always)]
115 unsafe fn and_not(self, rhs: Self) -> Self {
116 self_from_op!(_mm_andnot_si128, rhs, self)
117 }
118}
119
120macro_rules! scalar_shift {
121 ($self:expr, $rhs:expr, $lanes:expr, $elem:ty, $uelem:ty, $op:tt) => {{
122 let mut a = [<$elem>::default(); $lanes];
123 let mut b = [<$elem>::default(); $lanes];
124 _mm_storeu_si128(a.as_mut_ptr() as *mut __m128i, $self.0);
125 _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, $rhs.0);
126 let width = (std::mem::size_of::<$elem>() * 8) as u32;
127 let mut out = [<$elem>::default(); $lanes];
128 for i in 0..$lanes {
129 let shift = b[i] as $uelem as u32;
130 out[i] = scalar_shift!(@apply $op, a[i], shift, width, $elem, $uelem);
131 }
132 Self(_mm_loadu_si128(out.as_ptr() as *const __m128i))
133 }};
134 (@apply sll, $val:expr, $shift:expr, $width:expr, $elem:ty, $uelem:ty) => {
135 if $shift >= $width { 0 } else { (($val as $uelem) << $shift) as $elem }
136 };
137 (@apply srl, $val:expr, $shift:expr, $width:expr, $elem:ty, $uelem:ty) => {
138 if $shift >= $width { 0 } else { (($val as $uelem) >> $shift) as $elem }
139 };
140 (@apply sra, $val:expr, $shift:expr, $width:expr, $elem:ty, $uelem:ty) => {
141 $val >> $shift.min($width - 1)
142 };
143}
144
145impl SimdShiftImpl for SseReg {
146 #[inline(always)]
147 unsafe fn sllv_64(self, rhs: Self) -> Self {
148 unsafe { scalar_shift!(self, rhs, 2, i64, u64, sll) }
149 }
150 #[inline(always)]
151 unsafe fn srlv_64(self, rhs: Self) -> Self {
152 unsafe { scalar_shift!(self, rhs, 2, i64, u64, srl) }
153 }
154 #[inline(always)]
155 unsafe fn srav_64(self, rhs: Self) -> Self {
156 unsafe { scalar_shift!(self, rhs, 2, i64, u64, sra) }
157 }
158 #[inline(always)]
159 unsafe fn sllv_32(self, rhs: Self) -> Self {
160 unsafe { scalar_shift!(self, rhs, 4, i32, u32, sll) }
161 }
162 #[inline(always)]
163 unsafe fn srlv_32(self, rhs: Self) -> Self {
164 unsafe { scalar_shift!(self, rhs, 4, i32, u32, srl) }
165 }
166 #[inline(always)]
167 unsafe fn srav_32(self, rhs: Self) -> Self {
168 unsafe { scalar_shift!(self, rhs, 4, i32, u32, sra) }
169 }
170 #[inline(always)]
171 unsafe fn sllv_16(self, rhs: Self) -> Self {
172 unsafe { scalar_shift!(self, rhs, 8, i16, u16, sll) }
173 }
174 #[inline(always)]
175 unsafe fn srlv_16(self, rhs: Self) -> Self {
176 unsafe { scalar_shift!(self, rhs, 8, i16, u16, srl) }
177 }
178 #[inline(always)]
179 unsafe fn srav_16(self, rhs: Self) -> Self {
180 unsafe { scalar_shift!(self, rhs, 8, i16, u16, sra) }
181 }
182}
183impl SimdLoadImpl for SseReg {
184 type MaskType = Self;
185
186 #[inline(always)]
187 unsafe fn load_aligned<T>(ptr: *const T) -> Self {
188 self_from_op!(_mm_load_si128, ptr)
189 }
190 #[inline(always)]
191 unsafe fn load_unaligned<T>(ptr: *const T) -> Self {
192 self_from_op!(_mm_loadu_si128, ptr)
193 }
194
195 #[inline(always)]
196 unsafe fn masked_load_64<T>(ptr: *const T, mask: Self::MaskType) -> Self {
197 unsafe {
198 let mut m = [0i64; 2];
199 _mm_storeu_si128(m.as_mut_ptr() as *mut __m128i, mask.0);
200 let src = ptr as *const i64;
201 let mut out = [0i64; 2];
202 for i in 0..2 {
203 if (m[i] as u64) & (1 << 63) != 0 {
205 out[i] = *src.add(i);
206 }
207 }
208 Self(_mm_loadu_si128(out.as_ptr() as *const __m128i))
209 }
210 }
211
212 #[inline(always)]
213 unsafe fn masked_load_32<T>(ptr: *const T, mask: Self::MaskType) -> Self {
214 unsafe {
215 let mut m = [0i32; 4];
216 _mm_storeu_si128(m.as_mut_ptr() as *mut __m128i, mask.0);
217 let src = ptr as *const i32;
218 let mut out = [0i32; 4];
219 for i in 0..4 {
220 if (m[i] as u32) & (1 << 31) != 0 {
221 out[i] = *src.add(i);
222 }
223 }
224 Self(_mm_loadu_si128(out.as_ptr() as *const __m128i))
225 }
226 }
227}
228
229impl SimdStoreImpl for SseReg {
230 type MaskType = Self;
231
232 #[inline(always)]
233 unsafe fn store_aligned<T>(self, ptr: *mut T) {
234 execute_intrinsic!(_mm_store_si128, ptr, self);
235 }
236 #[inline(always)]
237 unsafe fn store_unaligned<T>(self, ptr: *mut T) {
238 execute_intrinsic!(_mm_storeu_si128, ptr, self);
239 }
240
241 #[inline(always)]
242 unsafe fn masked_store_64<T>(self, ptr: *mut T, mask: Self::MaskType) {
243 unsafe {
244 let mut m = [0i64; 2];
245 let mut v = [0i64; 2];
246 _mm_storeu_si128(m.as_mut_ptr() as *mut __m128i, mask.0);
247 _mm_storeu_si128(v.as_mut_ptr() as *mut __m128i, self.0);
248 let dst = ptr as *mut i64;
249 for i in 0..2 {
250 if (m[i] as u64) & (1 << 63) != 0 {
251 *dst.add(i) = v[i];
252 }
253 }
254 }
255 }
256
257 #[inline(always)]
258 unsafe fn masked_store_32<T>(self, ptr: *mut T, mask: Self::MaskType) {
259 unsafe {
260 let mut m = [0i32; 4];
261 let mut v = [0i32; 4];
262 _mm_storeu_si128(m.as_mut_ptr() as *mut __m128i, mask.0);
263 _mm_storeu_si128(v.as_mut_ptr() as *mut __m128i, self.0);
264 let dst = ptr as *mut i32;
265 for i in 0..4 {
266 if (m[i] as u32) & (1 << 31) != 0 {
267 *dst.add(i) = v[i];
268 }
269 }
270 }
271 }
272}
273
274impl SimdZeroImpl for SseReg {
275 #[inline(always)]
276 unsafe fn zero() -> Self {
277 self_from_op!(_mm_setzero_si128,)
278 }
279}
280
281impl SimdFloatCastsImpl for SseReg {
282 #[inline(always)]
283 unsafe fn float_to_int_trunc(self) -> Self {
284 self_from_op!(_mm_cvttps_epi32, self)
285 }
286 #[inline(always)]
287 unsafe fn float_to_int_round(self) -> Self {
288 self_from_op!(_mm_cvtps_epi32, self)
289 }
290}
291
292impl SimdIntCastsImpl for SseReg {
293 #[inline(always)]
294 unsafe fn int_to_float(self) -> Self {
295 self_from_op!(_mm_cvtepi32_ps, self)
296 }
297}
298
299impl SimdPermuteImpl for SseReg {
300 #[inline(always)]
301 unsafe fn permute_32(self, rhs: Self) -> Self {
302 unsafe {
303 let mut a = [0.0f32; 4];
304 let mut idx = [0i32; 4];
305 _mm_storeu_ps(a.as_mut_ptr(), std::mem::transmute(self.0));
306 _mm_storeu_si128(idx.as_mut_ptr() as *mut __m128i, rhs.0);
307 let mut out = [0.0f32; 4];
308 for i in 0..4 {
309 out[i] = a[(idx[i] & 0b11) as usize]; }
311 Self(std::mem::transmute(_mm_loadu_ps(out.as_ptr())))
312 }
313 }
314
315 #[inline(always)]
316 unsafe fn permute_8(self, rhs: Self) -> Self {
317 self_from_op!(_mm_shuffle_epi8, self, rhs)
318 }
319}
320impl SimdVariableBlendImpl for SseReg {
321 type VecType = Self;
322 #[inline(always)]
323 unsafe fn vblend_64(self, true_values: Self::VecType, false_values: Self::VecType) -> Self {
324 self_from_op!(_mm_blendv_pd, false_values, true_values, self)
325 }
326 #[inline(always)]
327 unsafe fn vblend_32(self, true_values: Self::VecType, false_values: Self::VecType) -> Self {
328 self_from_op!(_mm_blendv_ps, false_values, true_values, self)
329 }
330 #[inline(always)]
331 unsafe fn vblend_8(self, true_values: Self::VecType, false_values: Self::VecType) -> Self {
332 self_from_op!(_mm_blendv_epi8, false_values, true_values, self)
333 }
334}
335
336impl SimdImmediateBlendImpl for SseReg {
337 #[inline(always)]
338 unsafe fn blend_64<const N: i32>(self, false_values: Self) -> Self {
339 self_from_const_op!(_mm_blend_pd, N, false_values, self)
340 }
341 #[inline(always)]
342 unsafe fn blend_32<const N: i32>(self, false_values: Self) -> Self {
343 self_from_const_op!(_mm_blend_ps, N, false_values, self)
344 }
345}
346
347impl SimdMulAddImpl for SseReg {
348 #[inline(always)]
349 unsafe fn mul_add_f64(self, mult: Self, add: Self) -> Self {
350 self_from_op!(_mm_add_pd, self_from_op!(_mm_mul_pd, self, mult), add)
351 }
352 #[inline(always)]
353 unsafe fn mul_sub_f64(self, mult: Self, sub: Self) -> Self {
354 self_from_op!(_mm_sub_pd, self_from_op!(_mm_mul_pd, self, mult), sub)
355 }
356 #[inline(always)]
357 unsafe fn negated_mul_add_f64(self, mult: Self, add: Self) -> Self {
358 self_from_op!(_mm_sub_pd, add, self_from_op!(_mm_mul_pd, self, mult))
359 }
360 #[inline(always)]
361 unsafe fn negated_mul_sub_f64(self, mult: Self, sub: Self) -> Self {
362 let neg = self_from_op!(_mm_mul_pd, self, mult);
363 self_from_op!(
364 _mm_sub_pd,
365 self_from_op!(_mm_xor_pd, neg, Self::splat_64(-0.0f64).0),
366 sub
367 )
368 }
369
370 #[inline(always)]
371 unsafe fn mul_add_f32(self, mult: Self, add: Self) -> Self {
372 self_from_op!(_mm_add_ps, self_from_op!(_mm_mul_ps, self, mult), add)
373 }
374 #[inline(always)]
375 unsafe fn mul_sub_f32(self, mult: Self, sub: Self) -> Self {
376 self_from_op!(_mm_sub_ps, self_from_op!(_mm_mul_ps, self, mult), sub)
377 }
378 #[inline(always)]
379 unsafe fn negated_mul_add_f32(self, mult: Self, add: Self) -> Self {
380 self_from_op!(_mm_sub_ps, add, self_from_op!(_mm_mul_ps, self, mult))
381 }
382 #[inline(always)]
383 unsafe fn negated_mul_sub_f32(self, mult: Self, sub: Self) -> Self {
384 let neg = self_from_op!(_mm_mul_ps, self, mult);
385 self_from_op!(
386 _mm_sub_ps,
387 self_from_op!(_mm_xor_ps, neg, Self::splat_32(-0.0f32).0),
388 sub
389 )
390 }
391}
392
393impl SimdRoundImpl for SseReg {
394 #[inline(always)]
395 unsafe fn round_f64(self) -> Self {
396 self_from_const_op!(
397 _mm_round_pd,
398 _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC,
399 self
400 )
401 }
402 #[inline(always)]
403 unsafe fn round_f32(self) -> Self {
404 self_from_const_op!(
405 _mm_round_ps,
406 _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC,
407 self
408 )
409 }
410 #[inline(always)]
411 unsafe fn floor_f64(self) -> Self {
412 self_from_const_op!(
413 _mm_round_pd,
414 _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC,
415 self
416 )
417 }
418 #[inline(always)]
419 unsafe fn floor_f32(self) -> Self {
420 self_from_const_op!(
421 _mm_round_ps,
422 _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC,
423 self
424 )
425 }
426 #[inline(always)]
427 unsafe fn ceil_f64(self) -> Self {
428 self_from_const_op!(
429 _mm_round_pd,
430 _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC,
431 self
432 )
433 }
434 #[inline(always)]
435 unsafe fn ceil_f32(self) -> Self {
436 self_from_const_op!(
437 _mm_round_ps,
438 _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC,
439 self
440 )
441 }
442}
443
444impl SimdPartialOrdImpl for SseReg {
445 type MaskType = Self;
446 #[inline(always)]
447 unsafe fn cmp_f64_eq(self, rhs: Self) -> Self {
448 self_from_op!(_mm_cmpeq_pd, self, rhs)
449 }
450 #[inline(always)]
451 unsafe fn cmp_f64_lt(self, rhs: Self) -> Self {
452 self_from_op!(_mm_cmplt_pd, self, rhs)
453 }
454 #[inline(always)]
455 unsafe fn cmp_f64_le(self, rhs: Self) -> Self {
456 self_from_op!(_mm_cmple_pd, self, rhs)
457 }
458 #[inline(always)]
459 unsafe fn cmp_f64_gt(self, rhs: Self) -> Self {
460 self_from_op!(_mm_cmpgt_pd, self, rhs)
461 }
462 #[inline(always)]
463 unsafe fn cmp_f64_ge(self, rhs: Self) -> Self {
464 self_from_op!(_mm_cmpge_pd, self, rhs)
465 }
466 #[inline(always)]
467 unsafe fn cmp_f64_neq(self, rhs: Self) -> Self {
468 self_from_op!(_mm_cmpneq_pd, self, rhs)
469 }
470 #[inline(always)]
471 unsafe fn cmp_f32_eq(self, rhs: Self) -> Self {
472 self_from_op!(_mm_cmpeq_ps, self, rhs)
473 }
474 #[inline(always)]
475 unsafe fn cmp_f32_lt(self, rhs: Self) -> Self {
476 self_from_op!(_mm_cmplt_ps, self, rhs)
477 }
478 #[inline(always)]
479 unsafe fn cmp_f32_le(self, rhs: Self) -> Self {
480 self_from_op!(_mm_cmple_ps, self, rhs)
481 }
482 #[inline(always)]
483 unsafe fn cmp_f32_gt(self, rhs: Self) -> Self {
484 self_from_op!(_mm_cmpgt_ps, self, rhs)
485 }
486 #[inline(always)]
487 unsafe fn cmp_f32_ge(self, rhs: Self) -> Self {
488 self_from_op!(_mm_cmpge_ps, self, rhs)
489 }
490 #[inline(always)]
491 unsafe fn cmp_f32_neq(self, rhs: Self) -> Self {
492 self_from_op!(_mm_cmpneq_ps, self, rhs)
493 }
494 #[inline(always)]
495 unsafe fn cmp_i64_eq(self, rhs: Self) -> Self {
496 self_from_op!(_mm_cmpeq_epi64, self, rhs)
497 }
498 #[inline(always)]
499 unsafe fn cmp_i64_gt(self, rhs: Self) -> Self {
500 self_from_op!(_mm_cmpgt_epi64, self, rhs)
501 }
502 #[inline(always)]
503 unsafe fn cmp_i32_eq(self, rhs: Self) -> Self {
504 self_from_op!(_mm_cmpeq_epi32, self, rhs)
505 }
506 #[inline(always)]
507 unsafe fn cmp_i32_gt(self, rhs: Self) -> Self {
508 self_from_op!(_mm_cmpgt_epi32, self, rhs)
509 }
510 #[inline(always)]
511 unsafe fn cmp_i16_eq(self, rhs: Self) -> Self {
512 self_from_op!(_mm_cmpeq_epi16, self, rhs)
513 }
514 #[inline(always)]
515 unsafe fn cmp_i16_gt(self, rhs: Self) -> Self {
516 self_from_op!(_mm_cmpgt_epi16, self, rhs)
517 }
518 #[inline(always)]
519 unsafe fn cmp_i8_eq(self, rhs: Self) -> Self {
520 self_from_op!(_mm_cmpeq_epi8, self, rhs)
521 }
522 #[inline(always)]
523 unsafe fn cmp_i8_gt(self, rhs: Self) -> Self {
524 self_from_op!(_mm_cmpgt_epi8, self, rhs)
525 }
526
527 #[inline(always)]
528 unsafe fn max_f64(self, rhs: Self) -> Self {
529 self_from_op!(_mm_max_pd, self, rhs)
530 }
531 #[inline(always)]
532 unsafe fn min_f64(self, rhs: Self) -> Self {
533 self_from_op!(_mm_min_pd, self, rhs)
534 }
535 #[inline(always)]
536 unsafe fn max_f32(self, rhs: Self) -> Self {
537 self_from_op!(_mm_max_ps, self, rhs)
538 }
539 #[inline(always)]
540 unsafe fn min_f32(self, rhs: Self) -> Self {
541 self_from_op!(_mm_min_ps, self, rhs)
542 }
543 #[inline(always)]
544 unsafe fn max_i32(self, rhs: Self) -> Self {
545 self_from_op!(_mm_max_epi32, self, rhs)
546 }
547 #[inline(always)]
548 unsafe fn min_i32(self, rhs: Self) -> Self {
549 self_from_op!(_mm_min_epi32, self, rhs)
550 }
551 #[inline(always)]
552 unsafe fn max_i16(self, rhs: Self) -> Self {
553 self_from_op!(_mm_max_epi16, self, rhs)
554 }
555 #[inline(always)]
556 unsafe fn min_i16(self, rhs: Self) -> Self {
557 self_from_op!(_mm_min_epi16, self, rhs)
558 }
559 #[inline(always)]
560 unsafe fn max_i8(self, rhs: Self) -> Self {
561 self_from_op!(_mm_max_epi8, self, rhs)
562 }
563 #[inline(always)]
564 unsafe fn min_i8(self, rhs: Self) -> Self {
565 self_from_op!(_mm_min_epi8, self, rhs)
566 }
567 #[inline(always)]
568 unsafe fn max_u32(self, rhs: Self) -> Self {
569 self_from_op!(_mm_max_epu32, self, rhs)
570 }
571 #[inline(always)]
572 unsafe fn min_u32(self, rhs: Self) -> Self {
573 self_from_op!(_mm_min_epu32, self, rhs)
574 }
575 #[inline(always)]
576 unsafe fn max_u16(self, rhs: Self) -> Self {
577 self_from_op!(_mm_max_epu16, self, rhs)
578 }
579 #[inline(always)]
580 unsafe fn min_u16(self, rhs: Self) -> Self {
581 self_from_op!(_mm_min_epu16, self, rhs)
582 }
583 #[inline(always)]
584 unsafe fn max_u8(self, rhs: Self) -> Self {
585 self_from_op!(_mm_max_epu8, self, rhs)
586 }
587 #[inline(always)]
588 unsafe fn min_u8(self, rhs: Self) -> Self {
589 self_from_op!(_mm_min_epu8, self, rhs)
590 }
591}
592
593impl SimdSplatImpl for SseReg {
595 #[inline(always)]
596 unsafe fn splat_64<T>(val: T) -> Self {
597 self_from_op!(_mm_set1_epi64x, val)
598 }
599 #[inline(always)]
600 unsafe fn splat_32<T>(val: T) -> Self {
601 self_from_op!(_mm_set1_epi32, val)
602 }
603 #[inline(always)]
604 unsafe fn splat_16<T>(val: T) -> Self {
605 self_from_op!(_mm_set1_epi16, val)
606 }
607 #[inline(always)]
608 unsafe fn splat_8<T>(val: T) -> Self {
609 self_from_op!(_mm_set1_epi8, val)
610 }
611}
612
613impl SimdGatherImpl for SseReg {
614 #[inline(always)]
615 unsafe fn gather_32_from_32<T, const B: i32>(self, ptr: *const T) -> Self {
616 unsafe {
617 let mut idx = [0i32; 4];
618 _mm_storeu_si128(idx.as_mut_ptr() as *mut __m128i, self.0);
619 let base = ptr as *const u8;
620 let mut out = [0i32; 4];
621 for i in 0..4 {
622 let byte_offset = (idx[i] as isize) * (B as isize);
623 out[i] = *(base.offset(byte_offset) as *const i32);
624 }
625 Self(_mm_loadu_si128(out.as_ptr() as *const __m128i))
626 }
627 }
628
629 #[inline(always)]
630 unsafe fn gather_64_from_64<T, const B: i32>(self, ptr: *const T) -> Self {
631 unsafe {
632 let mut idx = [0i64; 2];
633 _mm_storeu_si128(idx.as_mut_ptr() as *mut __m128i, self.0);
634 let base = ptr as *const u8;
635 let mut out = [0i64; 2];
636 for i in 0..2 {
637 let byte_offset = (idx[i] as isize) * (B as isize);
638 out[i] = *(base.offset(byte_offset) as *const i64);
639 }
640 Self(_mm_loadu_si128(out.as_ptr() as *const __m128i))
641 }
642 }
643}
644
645impl SimdSqrtImpl for SseReg {
646 #[inline(always)]
647 unsafe fn sqrt_f64(self) -> Self {
648 self_from_op!(_mm_sqrt_pd, self)
649 }
650 #[inline(always)]
651 unsafe fn sqrt_f32(self) -> Self {
652 self_from_op!(_mm_sqrt_ps, self)
653 }
654 #[inline(always)]
655 unsafe fn rsqrt_f32(self) -> Self {
656 self_from_op!(_mm_rsqrt_ps, self)
657 }
658}
659
660impl SimdAllBitsImpl for SseReg {
661 #[inline(always)]
662 unsafe fn all_zero(self) -> bool {
663 execute_intrinsic!(_mm_testz_si128, self, self) != 0
664 }
665}
666
667impl SimdNegateImpl for SseReg {
668 #[inline(always)]
669 unsafe fn negate_f64(self) -> Self {
670 unsafe { Self::splat_64(-0.0f64).xor(self) }
671 }
672 #[inline(always)]
673 unsafe fn negate_f32(self) -> Self {
674 unsafe { Self::splat_32(-0.0f64).xor(self) }
675 }
676}
677
678impl SimdBlockShiftImpl for SseReg {
680 #[inline(always)]
681 unsafe fn block_left_byte_shift<const N: i32>(self) -> Self {
682 self_from_const_op!(_mm_bslli_si128, N, self)
683 }
684 #[inline(always)]
685 unsafe fn block_right_byte_shift<const N: i32>(self) -> Self {
686 self_from_const_op!(_mm_bsrli_si128, N, self)
687 }
688}
689
690impl SimdMaskBitConversion for SseReg {
691 #[inline(always)]
692 unsafe fn to_bits_64(self) -> u64 {
693 execute_intrinsic!(_mm_movemask_pd, self) as u64
694 }
695 #[inline(always)]
696 unsafe fn to_bits_32(self) -> u64 {
697 execute_intrinsic!(_mm_movemask_ps, self) as u64
698 }
699 #[inline(always)]
700 unsafe fn to_bits_8(self) -> u64 {
701 execute_intrinsic!(_mm_movemask_epi8, self) as u64
702 }
703 #[inline(always)]
704 unsafe fn from_bits_64(bitmask: u64) -> Self {
705 let mask = self_from_op!(_mm_set_epi64x, 1, 2);
706 let bits = self_from_op!(_mm_set1_epi64x, bitmask as i64);
707 self_from_op!(_mm_cmpgt_epi64, bits.and(mask), Self::zero()) }
709 #[inline(always)]
710 unsafe fn from_bits_32(bitmask: u64) -> Self {
711 let mask = self_from_op!(_mm_set_epi32, 1, 2, 4, 8);
712 let bits = self_from_op!(_mm_set1_epi32, bitmask as u32);
713 self_from_op!(_mm_cmpgt_epi32, bits.and(mask), Self::zero())
714 }
715 #[inline(always)]
716 unsafe fn from_bits_16(bitmask: u64) -> Self {
717 #[rustfmt::skip]
718 let mask = self_from_op!(_mm_set_epi16, 1, 2, 4, 8, 16, 32, 64, 128);
719 let bits = self_from_op!(_mm_set1_epi16, bitmask as i16);
720 unsafe { self_from_op!(_mm_cmpeq_epi16, bits.and(mask), Self::zero()).not() }
721 }
722 #[inline(always)]
723 unsafe fn from_bits_8(bitmask: u64) -> Self {
724 let b1 = bitmask as i8;
725 let b2 = (bitmask >> 8) as i8;
726 #[rustfmt::skip]
727 let mask = self_from_op!(_mm_set_epi8,
728 1, 2, 4, 8, 16, 32, 64, -128,
729 1, 2, 4, 8, 16, 32, 64, -128
730 );
731 #[rustfmt::skip]
732 let bits = self_from_op!(_mm_set_epi8,
733 b1, b1, b1, b1, b1, b1, b1, b1,
734 b2, b2, b2, b2, b2, b2, b2, b2
735 );
736 unsafe { self_from_op!(_mm_cmpeq_epi8, bits.and(mask), Self::zero()).not() }
737 }
738}
739
740impl SimdLaneShiftImpl for SseReg {
741 #[inline(always)]
742 unsafe fn left_lane_shift_32(self, n: u32) -> Self {
743 match n {
744 0 => self,
745 1 => self_from_const_op!(_mm_bsrli_si128, 4, self),
746 2 => self_from_const_op!(_mm_bsrli_si128, 8, self),
747 3 => self_from_const_op!(_mm_bsrli_si128, 12, self),
748 _ => unsafe { Self::zero() },
749 }
750 }
751 #[inline(always)]
752 unsafe fn right_lane_shift_32(self, n: u32) -> Self {
753 match n {
754 0 => self,
755 1 => self_from_const_op!(_mm_bslli_si128, 4, self),
756 2 => self_from_const_op!(_mm_bslli_si128, 8, self),
757 3 => self_from_const_op!(_mm_bslli_si128, 12, self),
758 _ => unsafe { Self::zero() },
759 }
760 }
761}