Skip to main content

simply_simd/architectures/intrinsics/
scalar.rs

1use std::mem::transmute_copy;
2
3use crate::architectures::interface::*;
4
5#[derive(Copy, Clone)]
6#[repr(align(8))]
7pub struct ScalarReg<const N: usize>(pub [u8; N]);
8#[derive(Copy, Clone)]
9pub struct ScalarMask<const N: usize>(pub [bool; N]);
10
11impl<const N: usize> SimdArch for ScalarReg<N> {}
12impl<const N: usize> MaskArch for ScalarMask<N> {}
13
14macro_rules! scalar_token_op {
15    ($type:ty, $op:tt, $self:ident, $rhs:ident, $size:expr) => {
16        unsafe {
17            let mut new = ScalarReg::<$size>([0; $size]);
18
19            let self_ptr: *const $type = $self.0.as_ptr() as *const $type;
20            let rhs_ptr: *const $type = $rhs.0.as_ptr() as *const $type;
21            let new_ptr: *mut $type = new.0.as_mut_ptr() as *mut $type;
22
23            for i in 0..($size / size_of::<$type>()) {
24                *new_ptr.add(i) = *self_ptr.add(i) $op *rhs_ptr.add(i);
25            }
26
27            new
28        }
29    }
30}
31
32macro_rules! scalar_token_op_usize_rhs {
33    ($type:ty, $op:tt, $self:ident, $rhs:ident, $size:expr) => {
34        unsafe {
35            let mut new = ScalarReg::<$size>([0; $size]);
36
37            let self_ptr: *const $type = $self.0.as_ptr() as *const $type;
38            let rhs_ptr: *const $type = $rhs.0.as_ptr() as *const $type;
39            let new_ptr: *mut $type = new.0.as_mut_ptr() as *mut $type;
40
41            for i in 0..($size / size_of::<$type>()) {
42                *new_ptr.add(i) = *self_ptr.add(i) $op (*rhs_ptr.add(i) as usize);
43            }
44
45            new
46        }
47    }
48}
49
50macro_rules! scalar_func_op {
51    ($type:ty, $op:ident, $self:ident, $rhs:ident, $size:expr) => {
52        unsafe {
53            let mut new = ScalarReg::<$size>([0; $size]);
54
55            let self_ptr: *const $type = $self.0.as_ptr() as *const $type;
56            let rhs_ptr: *const $type = $rhs.0.as_ptr() as *const $type;
57            let new_ptr: *mut $type = new.0.as_mut_ptr() as *mut $type;
58
59            for i in 0..($size / size_of::<$type>()) {
60                *new_ptr.add(i) = (*self_ptr.add(i)).$op(*rhs_ptr.add(i));
61            }
62
63            new
64        }
65    };
66}
67
68macro_rules! scalar_self_op {
69    ($type:ty, $op:ident, $self:ident, $size:expr) => {
70        unsafe {
71            let mut new = ScalarReg::<$size>([0; $size]);
72
73            let self_ptr: *const $type = $self.0.as_ptr() as *const $type;
74            let new_ptr: *mut $type = new.0.as_mut_ptr() as *mut $type;
75
76            for i in 0..($size / size_of::<$type>()) {
77                *new_ptr.add(i) = (*self_ptr.add(i)).$op();
78            }
79            new
80        }
81    };
82}
83
84macro_rules! scalar_fma_expr_op {
85    ($type:ty, $self:ident, $mult:ident, $add:ident, $size:expr, |$a:ident, $b:ident, $c:ident| $op:expr) => {
86        unsafe {
87            let mut new = ScalarReg::<$size>([0; $size]);
88
89            let a_ptr: *const $type = $self.0.as_ptr() as *const $type;
90            let b_ptr: *const $type = $mult.0.as_ptr() as *const $type;
91            let c_ptr: *const $type = $add.0.as_ptr() as *const $type;
92            let new_ptr: *mut $type = new.0.as_mut_ptr() as *mut $type;
93
94            for i in 0..($size / size_of::<$type>()) {
95                let $a = *a_ptr.add(i);
96                let $b = *b_ptr.add(i);
97                let $c = *c_ptr.add(i);
98                *new_ptr.add(i) = $op;
99            }
100            new
101        }
102    };
103}
104
105macro_rules! scalar_cmp {
106    {$type:ty, $op:tt, $self:ident, $rhs:ident, $size:expr} => {
107        unsafe {
108            let mut result = ScalarMask::<$size>([false; $size]);
109
110            let self_ptr = $self.0.as_ptr() as *const $type;
111            let rhs_ptr = $rhs.0.as_ptr() as *const $type;
112
113            for i in 0..($size / size_of::<$type>()) {
114                result.0[i] = *self_ptr.add(i) $op *rhs_ptr.add(i);
115            }
116
117            result
118        }
119    }
120}
121
122macro_rules! scalar_splat {
123    {$type:ty, $self:ident, $val:expr, $size:expr} => {
124        unsafe {
125            let mut new = ScalarReg::<$size>([0; $size]);
126            let new_ptr = new.0.as_mut_ptr() as *mut $type;
127
128            for i in 0..(N / size_of::<$type>()) {
129                *new_ptr.add(i) = $val;
130            }
131
132            new
133        }
134    }
135}
136
137impl<const N: usize> SimdAddImpl for ScalarReg<N> {
138    #[inline(always)]
139    unsafe fn f64_add(self, rhs: Self) -> Self {
140        scalar_token_op!(f64, +, self, rhs, N)
141    }
142    #[inline(always)]
143    unsafe fn f32_add(self, rhs: Self) -> Self {
144        scalar_token_op!(f32, +, self, rhs, N)
145    }
146    #[inline(always)]
147    unsafe fn i64_add(self, rhs: Self) -> Self {
148        scalar_func_op!(i64, wrapping_add, self, rhs, N)
149    }
150    #[inline(always)]
151    unsafe fn i32_add(self, rhs: Self) -> Self {
152        scalar_func_op!(i32, wrapping_add, self, rhs, N)
153    }
154    #[inline(always)]
155    unsafe fn i16_add(self, rhs: Self) -> Self {
156        scalar_func_op!(i16, wrapping_add, self, rhs, N)
157    }
158    #[inline(always)]
159    unsafe fn i8_add(self, rhs: Self) -> Self {
160        scalar_func_op!(i8, wrapping_add, self, rhs, N)
161    }
162}
163
164impl<const N: usize> SimdSubImpl for ScalarReg<N> {
165    #[inline(always)]
166    unsafe fn f64_sub(self, rhs: Self) -> Self {
167        scalar_token_op!(f64, -, self, rhs, N)
168    }
169    #[inline(always)]
170    unsafe fn f32_sub(self, rhs: Self) -> Self {
171        scalar_token_op!(f32, -, self, rhs, N)
172    }
173    #[inline(always)]
174    unsafe fn i64_sub(self, rhs: Self) -> Self {
175        scalar_func_op!(i64, wrapping_sub, self, rhs, N)
176    }
177    #[inline(always)]
178    unsafe fn i32_sub(self, rhs: Self) -> Self {
179        scalar_func_op!(i32, wrapping_sub, self, rhs, N)
180    }
181    #[inline(always)]
182    unsafe fn i16_sub(self, rhs: Self) -> Self {
183        scalar_func_op!(i16, wrapping_sub, self, rhs, N)
184    }
185    #[inline(always)]
186    unsafe fn i8_sub(self, rhs: Self) -> Self {
187        scalar_func_op!(i8, wrapping_sub, self, rhs, N)
188    }
189}
190
191impl<const N: usize> SimdMulImpl for ScalarReg<N> {
192    #[inline(always)]
193    unsafe fn f64_mul(self, rhs: Self) -> Self {
194        scalar_token_op!(f64, *, self, rhs, N)
195    }
196    #[inline(always)]
197    unsafe fn f32_mul(self, rhs: Self) -> Self {
198        scalar_token_op!(f32, *, self, rhs, N)
199    }
200    #[inline(always)]
201    unsafe fn i32_mul(self, rhs: Self) -> Self {
202        scalar_func_op!(i32, wrapping_mul, self, rhs, N)
203    }
204    #[inline(always)]
205    unsafe fn i16_mul(self, rhs: Self) -> Self {
206        scalar_func_op!(i16, wrapping_mul, self, rhs, N)
207    }
208}
209
210impl<const N: usize> SimdDivImpl for ScalarReg<N> {
211    #[inline(always)]
212    unsafe fn f64_div(self, rhs: Self) -> Self {
213        scalar_token_op!(f64, /, self, rhs, N)
214    }
215    #[inline(always)]
216    unsafe fn f32_div(self, rhs: Self) -> Self {
217        scalar_token_op!(f32, /, self, rhs, N)
218    }
219}
220
221impl<const N: usize> SimdBitwiseImpl for ScalarReg<N> {
222    #[inline(always)]
223    unsafe fn and(self, rhs: Self) -> Self {
224        scalar_token_op!(u64, &, self, rhs, N)
225    }
226    #[inline(always)]
227    unsafe fn or(self, rhs: Self) -> Self {
228        scalar_token_op!(u64, |, self, rhs, N)
229    }
230    #[inline(always)]
231    unsafe fn xor(self, rhs: Self) -> Self {
232        scalar_token_op!(u64, ^, self, rhs, N)
233    }
234    #[inline(always)]
235    unsafe fn not(self) -> Self {
236        unsafe { self.xor(Self([255; N])) }
237    }
238    #[inline(always)]
239    unsafe fn and_not(self, rhs: Self) -> Self {
240        unsafe { self.and(rhs.not()) }
241    }
242}
243
244impl<const N: usize> SimdShiftImpl for ScalarReg<N> {
245    #[inline(always)]
246    unsafe fn sllv_64(self, rhs: Self) -> Self {
247        scalar_token_op_usize_rhs!(u64, <<, self, rhs, N)
248    }
249    #[inline(always)]
250    unsafe fn srlv_64(self, rhs: Self) -> Self {
251        scalar_token_op_usize_rhs!(u64, >>, self, rhs, N)
252    }
253    #[inline(always)]
254    unsafe fn srav_64(self, rhs: Self) -> Self {
255        scalar_token_op_usize_rhs!(i64, >>, self, rhs, N)
256    }
257    #[inline(always)]
258    unsafe fn sllv_32(self, rhs: Self) -> Self {
259        scalar_token_op_usize_rhs!(u32, <<, self, rhs, N)
260    }
261    #[inline(always)]
262    unsafe fn srlv_32(self, rhs: Self) -> Self {
263        scalar_token_op_usize_rhs!(u32, >>, self, rhs, N)
264    }
265    #[inline(always)]
266    unsafe fn srav_32(self, rhs: Self) -> Self {
267        scalar_token_op_usize_rhs!(i32, >>, self, rhs, N)
268    }
269    #[inline(always)]
270    unsafe fn sllv_16(self, rhs: Self) -> Self {
271        scalar_token_op_usize_rhs!(u16, <<, self, rhs, N)
272    }
273    #[inline(always)]
274    unsafe fn srlv_16(self, rhs: Self) -> Self {
275        scalar_token_op_usize_rhs!(u16, >>, self, rhs, N)
276    }
277    #[inline(always)]
278    unsafe fn srav_16(self, rhs: Self) -> Self {
279        scalar_token_op_usize_rhs!(i16, >>, self, rhs, N)
280    }
281}
282
283impl<const N: usize> SimdLoadImpl for ScalarReg<N> {
284    type MaskType = ScalarMask<N>;
285    #[inline(always)]
286    unsafe fn load_aligned<T>(ptr: *const T) -> Self {
287        unsafe {
288            let mut new = Self([0; N]);
289            std::ptr::copy_nonoverlapping(ptr as *const u8, new.0.as_mut_ptr(), N);
290            new
291        }
292    }
293    #[inline(always)]
294    unsafe fn load_unaligned<T>(ptr: *const T) -> Self {
295        unsafe { Self::load_aligned::<T>(ptr) }
296    }
297    #[inline(always)]
298    unsafe fn masked_load_64<T>(ptr: *const T, mask: Self::MaskType) -> Self {
299        unsafe {
300            let mut new = Self([0; N]);
301            let new_ptr = new.0.as_mut_ptr() as *mut u64;
302            let data_ptr = ptr as *const u64;
303            for i in 0..(N / 8) {
304                *new_ptr.add(i) = if mask.0[i] { *data_ptr.add(i) } else { 0 };
305            }
306            new
307        }
308    }
309    #[inline(always)]
310    unsafe fn masked_load_32<T>(ptr: *const T, mask: Self::MaskType) -> Self {
311        unsafe {
312            let mut new = Self([0; N]);
313            let new_ptr = new.0.as_mut_ptr() as *mut u32;
314            let data_ptr = ptr as *const u32;
315            for i in 0..(N / 4) {
316                *new_ptr.add(i) = if mask.0[i] { *data_ptr.add(i) } else { 0 };
317            }
318            new
319        }
320    }
321}
322
323impl<const N: usize> SimdStoreImpl for ScalarReg<N> {
324    type MaskType = ScalarMask<N>;
325    #[inline(always)]
326    unsafe fn store_aligned<T>(self, ptr: *mut T) {
327        unsafe {
328            std::ptr::copy_nonoverlapping(self.0.as_ptr(), ptr as *mut u8, N);
329        }
330    }
331    #[inline(always)]
332    unsafe fn store_unaligned<T>(self, ptr: *mut T) {
333        unsafe { self.store_aligned(ptr) };
334    }
335    #[inline(always)]
336    unsafe fn masked_store_64<T>(self, ptr: *mut T, mask: Self::MaskType) {
337        unsafe {
338            let self_ptr = self.0.as_ptr() as *const u64;
339            let store_ptr = ptr as *mut u64;
340            for i in 0..(N / 8) {
341                if mask.0[i] {
342                    *store_ptr.add(i) = *self_ptr.add(i);
343                }
344            }
345        }
346    }
347    #[inline(always)]
348    unsafe fn masked_store_32<T>(self, ptr: *mut T, mask: Self::MaskType) {
349        unsafe {
350            let self_ptr = self.0.as_ptr() as *const u32;
351            let store_ptr = ptr as *mut u32;
352            for i in 0..(N / 4) {
353                if mask.0[i] {
354                    *store_ptr.add(i) = *self_ptr.add(i);
355                }
356            }
357        }
358    }
359}
360
361impl<const N: usize> SimdZeroImpl for ScalarReg<N> {
362    #[inline(always)]
363    unsafe fn zero() -> Self {
364        Self([0; N])
365    }
366}
367
368impl<const N: usize> SimdFloatCastsImpl for ScalarReg<N> {
369    #[inline(always)]
370    unsafe fn float_to_int_trunc(self) -> Self {
371        unsafe {
372            let mut new = Self([0; N]);
373            let float_ptr = self.0.as_ptr() as *const f32;
374            let int_ptr = new.0.as_mut_ptr() as *mut i32;
375            for i in 0..(N / 4) {
376                *int_ptr.add(i) = (*float_ptr.add(i)) as i32;
377            }
378            new
379        }
380    }
381    #[inline(always)]
382    unsafe fn float_to_int_round(self) -> Self {
383        unsafe {
384            let mut new = Self([0; N]);
385            let float_ptr = self.0.as_ptr() as *const f32;
386            let int_ptr = new.0.as_mut_ptr() as *mut i32;
387            for i in 0..(N / 4) {
388                *int_ptr.add(i) = (*float_ptr.add(i)).round_ties_even() as i32;
389            }
390            new
391        }
392    }
393}
394
395impl<const N: usize> SimdIntCastsImpl for ScalarReg<N> {
396    #[inline(always)]
397    unsafe fn int_to_float(self) -> Self {
398        unsafe {
399            let mut new = Self([0; N]);
400            let int_ptr = self.0.as_ptr() as *const i32;
401            let float_ptr = new.0.as_mut_ptr() as *mut f32;
402            for i in 0..(N / 4) {
403                *float_ptr.add(i) = (*int_ptr.add(i)) as f32;
404            }
405            new
406        }
407    }
408}
409
410impl<const N: usize> SimdPermuteImpl for ScalarReg<N> {
411    #[inline(always)]
412    unsafe fn permute_32(self, rhs: Self) -> Self {
413        unsafe {
414            let mut new = Self([0; N]);
415            let new_ptr = new.0.as_mut_ptr() as *mut u32;
416            let self_ptr = self.0.as_ptr() as *const u32;
417            let indices_ptr = rhs.0.as_ptr() as *const u32;
418            for i in 0..(N / 4) {
419                let index = *indices_ptr.add(i) as usize;
420                *new_ptr.add(i) = if index < (N / 4) {
421                    *self_ptr.add(index)
422                } else {
423                    0
424                };
425            }
426            new
427        }
428    }
429    #[inline(always)]
430    unsafe fn permute_8(self, rhs: Self) -> Self {
431        unsafe {
432            let mut new = Self([0; N]);
433            let new_ptr = new.0.as_mut_ptr();
434            let self_ptr = self.0.as_ptr();
435            let indices_ptr = rhs.0.as_ptr();
436            for i in 0..N {
437                let lane_base = i & !15; // which 16-byte lane we're in (0 or 16)
438                let index = (*indices_ptr.add(i) as usize & 15) + lane_base;
439                *new_ptr.add(i) = *self_ptr.add(index);
440            }
441            new
442        }
443    }
444}
445
446impl<const N: usize> SimdVariableBlendImpl for ScalarMask<N> {
447    type VecType = ScalarReg<N>;
448    #[inline(always)]
449    unsafe fn vblend_64(
450        self,
451        true_values: Self::VecType,
452        false_values: Self::VecType,
453    ) -> ScalarReg<N> {
454        unsafe {
455            let mut new = ScalarReg::<N>([0; N]);
456            let new_ptr = new.0.as_mut_ptr() as *mut u64;
457            let false_ptr = false_values.0.as_ptr() as *const u64;
458            let true_ptr = true_values.0.as_ptr() as *const u64;
459            for i in 0..(N / 8) {
460                *new_ptr.add(i) = if self.0[i] {
461                    *true_ptr.add(i)
462                } else {
463                    *false_ptr.add(i)
464                };
465            }
466            new
467        }
468    }
469    #[inline(always)]
470    unsafe fn vblend_32(
471        self,
472        true_values: Self::VecType,
473        false_values: Self::VecType,
474    ) -> ScalarReg<N> {
475        unsafe {
476            let mut new = ScalarReg::<N>([0; N]);
477            let new_ptr = new.0.as_mut_ptr() as *mut u32;
478            let false_ptr = false_values.0.as_ptr() as *const u32;
479            let true_ptr = true_values.0.as_ptr() as *const u32;
480            for i in 0..(N / 4) {
481                *new_ptr.add(i) = if self.0[i] {
482                    *true_ptr.add(i)
483                } else {
484                    *false_ptr.add(i)
485                };
486            }
487            new
488        }
489    }
490    #[inline(always)]
491    unsafe fn vblend_8(
492        self,
493        true_values: Self::VecType,
494        false_values: Self::VecType,
495    ) -> ScalarReg<N> {
496        unsafe {
497            let mut new = ScalarReg::<N>([0; N]);
498            let new_ptr = new.0.as_mut_ptr();
499            let false_ptr = false_values.0.as_ptr();
500            let true_ptr = true_values.0.as_ptr();
501            for i in 0..N {
502                *new_ptr.add(i) = if self.0[i] {
503                    *true_ptr.add(i)
504                } else {
505                    *false_ptr.add(i)
506                };
507            }
508            new
509        }
510    }
511}
512
513impl<const M: usize> SimdImmediateBlendImpl for ScalarReg<M> {
514    #[inline(always)]
515    unsafe fn blend_64<const N: i32>(self, false_values: Self) -> Self {
516        unsafe {
517            let mut new = ScalarReg::<M>([0; M]);
518            let new_ptr = new.0.as_mut_ptr() as *mut u64;
519            let false_ptr = false_values.0.as_ptr() as *const u64;
520            let true_ptr = self.0.as_ptr() as *const u64;
521            for i in 0..(M >> 3) {
522                let cond = ((N >> i) & 1) == 1;
523                *new_ptr.add(i) = if cond {
524                    *true_ptr.add(i)
525                } else {
526                    *false_ptr.add(i)
527                };
528            }
529            new
530        }
531    }
532    #[inline(always)]
533    unsafe fn blend_32<const N: i32>(self, false_values: Self) -> Self {
534        unsafe {
535            let mut new = ScalarReg::<M>([0; M]);
536            let new_ptr = new.0.as_mut_ptr() as *mut u32;
537            let false_ptr = false_values.0.as_ptr() as *const u32;
538            let true_ptr = self.0.as_ptr() as *const u32;
539            for i in 0..(M >> 2) {
540                let cond = ((N >> i) & 1) == 1;
541                *new_ptr.add(i) = if cond {
542                    *true_ptr.add(i)
543                } else {
544                    *false_ptr.add(i)
545                };
546            }
547            new
548        }
549    }
550}
551
552impl<const N: usize> SimdMulAddImpl for ScalarReg<N> {
553    #[inline(always)]
554    unsafe fn mul_add_f64(self, mult: Self, add: Self) -> Self {
555        scalar_fma_expr_op!(f64, self, mult, add, N, |a, b, c| f64::mul_add(a, b, c))
556    }
557    #[inline(always)]
558    unsafe fn mul_sub_f64(self, mult: Self, sub: Self) -> Self {
559        scalar_fma_expr_op!(f64, self, mult, sub, N, |a, b, c| f64::mul_add(a, b, -c))
560    }
561    #[inline(always)]
562    unsafe fn negated_mul_add_f64(self, mult: Self, add: Self) -> Self {
563        scalar_fma_expr_op!(f64, self, mult, add, N, |a, b, c| f64::mul_add(-a, b, c))
564    }
565    #[inline(always)]
566    unsafe fn negated_mul_sub_f64(self, mult: Self, sub: Self) -> Self {
567        scalar_fma_expr_op!(f64, self, mult, sub, N, |a, b, c| f64::mul_add(-a, b, -c))
568    }
569    #[inline(always)]
570    unsafe fn mul_add_f32(self, mult: Self, add: Self) -> Self {
571        scalar_fma_expr_op!(f32, self, mult, add, N, |a, b, c| f32::mul_add(a, b, c))
572    }
573    #[inline(always)]
574    unsafe fn mul_sub_f32(self, mult: Self, sub: Self) -> Self {
575        scalar_fma_expr_op!(f32, self, mult, sub, N, |a, b, c| f32::mul_add(a, b, -c))
576    }
577    #[inline(always)]
578    unsafe fn negated_mul_add_f32(self, mult: Self, add: Self) -> Self {
579        scalar_fma_expr_op!(f32, self, mult, add, N, |a, b, c| f32::mul_add(-a, b, c))
580    }
581    #[inline(always)]
582    unsafe fn negated_mul_sub_f32(self, mult: Self, sub: Self) -> Self {
583        scalar_fma_expr_op!(f32, self, mult, sub, N, |a, b, c| f32::mul_add(-a, b, -c))
584    }
585}
586
587impl<const N: usize> SimdRoundImpl for ScalarReg<N> {
588    #[inline(always)]
589    unsafe fn round_f64(self) -> Self {
590        scalar_self_op!(f64, round_ties_even, self, N)
591    }
592    #[inline(always)]
593    unsafe fn round_f32(self) -> Self {
594        scalar_self_op!(f32, round_ties_even, self, N)
595    }
596    #[inline(always)]
597    unsafe fn floor_f64(self) -> Self {
598        scalar_self_op!(f64, floor, self, N)
599    }
600    #[inline(always)]
601    unsafe fn floor_f32(self) -> Self {
602        scalar_self_op!(f32, floor, self, N)
603    }
604    #[inline(always)]
605    unsafe fn ceil_f64(self) -> Self {
606        scalar_self_op!(f64, ceil, self, N)
607    }
608    #[inline(always)]
609    unsafe fn ceil_f32(self) -> Self {
610        scalar_self_op!(f32, ceil, self, N)
611    }
612}
613
614impl<const N: usize> SimdPartialOrdImpl for ScalarReg<N> {
615    type MaskType = ScalarMask<N>;
616    #[inline(always)]
617    unsafe fn cmp_f64_eq(self, rhs: Self) -> Self::MaskType {
618        scalar_cmp!(f64, ==, self, rhs, N)
619    }
620    #[inline(always)]
621    unsafe fn cmp_f64_lt(self, rhs: Self) -> Self::MaskType {
622        scalar_cmp!(f64, <, self, rhs, N)
623    }
624    #[inline(always)]
625    unsafe fn cmp_f64_le(self, rhs: Self) -> Self::MaskType {
626        scalar_cmp!(f64, <=, self, rhs, N)
627    }
628    #[inline(always)]
629    unsafe fn cmp_f64_gt(self, rhs: Self) -> Self::MaskType {
630        scalar_cmp!(f64, >, self, rhs, N)
631    }
632    #[inline(always)]
633    unsafe fn cmp_f64_ge(self, rhs: Self) -> Self::MaskType {
634        scalar_cmp!(f64, >=, self, rhs, N)
635    }
636    #[inline(always)]
637    unsafe fn cmp_f64_neq(self, rhs: Self) -> Self::MaskType {
638        scalar_cmp!(f64, !=, self, rhs, N)
639    }
640    #[inline(always)]
641    unsafe fn cmp_f32_eq(self, rhs: Self) -> Self::MaskType {
642        scalar_cmp!(f32, ==, self, rhs, N)
643    }
644    #[inline(always)]
645    unsafe fn cmp_f32_lt(self, rhs: Self) -> Self::MaskType {
646        scalar_cmp!(f32, <, self, rhs, N)
647    }
648    #[inline(always)]
649    unsafe fn cmp_f32_le(self, rhs: Self) -> Self::MaskType {
650        scalar_cmp!(f32, <=, self, rhs, N)
651    }
652    #[inline(always)]
653    unsafe fn cmp_f32_gt(self, rhs: Self) -> Self::MaskType {
654        scalar_cmp!(f32, >, self, rhs, N)
655    }
656    #[inline(always)]
657    unsafe fn cmp_f32_ge(self, rhs: Self) -> Self::MaskType {
658        scalar_cmp!(f32, >=, self, rhs, N)
659    }
660    #[inline(always)]
661    unsafe fn cmp_f32_neq(self, rhs: Self) -> Self::MaskType {
662        scalar_cmp!(f32, !=, self, rhs, N)
663    }
664    #[inline(always)]
665    unsafe fn cmp_i64_eq(self, rhs: Self) -> Self::MaskType {
666        scalar_cmp!(i64, ==, self, rhs, N)
667    }
668    #[inline(always)]
669    unsafe fn cmp_i64_gt(self, rhs: Self) -> Self::MaskType {
670        scalar_cmp!(i64, >, self, rhs, N)
671    }
672    #[inline(always)]
673    unsafe fn cmp_i32_eq(self, rhs: Self) -> Self::MaskType {
674        scalar_cmp!(i32, ==, self, rhs, N)
675    }
676    #[inline(always)]
677    unsafe fn cmp_i32_gt(self, rhs: Self) -> Self::MaskType {
678        scalar_cmp!(i32, >, self, rhs, N)
679    }
680    #[inline(always)]
681    unsafe fn cmp_i16_eq(self, rhs: Self) -> Self::MaskType {
682        scalar_cmp!(i16, ==, self, rhs, N)
683    }
684    #[inline(always)]
685    unsafe fn cmp_i16_gt(self, rhs: Self) -> Self::MaskType {
686        scalar_cmp!(i16, >, self, rhs, N)
687    }
688    #[inline(always)]
689    unsafe fn cmp_i8_eq(self, rhs: Self) -> Self::MaskType {
690        scalar_cmp!(i8, ==, self, rhs, N)
691    }
692    #[inline(always)]
693    unsafe fn cmp_i8_gt(self, rhs: Self) -> Self::MaskType {
694        scalar_cmp!(i8, >, self, rhs, N)
695    }
696
697    #[inline(always)]
698    unsafe fn max_f64(self, rhs: Self) -> Self {
699        scalar_func_op!(f64, max, self, rhs, N)
700    }
701    #[inline(always)]
702    unsafe fn min_f64(self, rhs: Self) -> Self {
703        scalar_func_op!(f64, min, self, rhs, N)
704    }
705    #[inline(always)]
706    unsafe fn max_f32(self, rhs: Self) -> Self {
707        scalar_func_op!(f32, max, self, rhs, N)
708    }
709    #[inline(always)]
710    unsafe fn min_f32(self, rhs: Self) -> Self {
711        scalar_func_op!(f32, min, self, rhs, N)
712    }
713    #[inline(always)]
714    unsafe fn max_i32(self, rhs: Self) -> Self {
715        scalar_func_op!(i32, max, self, rhs, N)
716    }
717    #[inline(always)]
718    unsafe fn min_i32(self, rhs: Self) -> Self {
719        scalar_func_op!(i32, min, self, rhs, N)
720    }
721    #[inline(always)]
722    unsafe fn max_i16(self, rhs: Self) -> Self {
723        scalar_func_op!(i16, max, self, rhs, N)
724    }
725    #[inline(always)]
726    unsafe fn min_i16(self, rhs: Self) -> Self {
727        scalar_func_op!(i16, min, self, rhs, N)
728    }
729    #[inline(always)]
730    unsafe fn max_i8(self, rhs: Self) -> Self {
731        scalar_func_op!(i8, max, self, rhs, N)
732    }
733    #[inline(always)]
734    unsafe fn min_i8(self, rhs: Self) -> Self {
735        scalar_func_op!(i8, min, self, rhs, N)
736    }
737    #[inline(always)]
738    unsafe fn max_u32(self, rhs: Self) -> Self {
739        scalar_func_op!(u32, max, self, rhs, N)
740    }
741    #[inline(always)]
742    unsafe fn min_u32(self, rhs: Self) -> Self {
743        scalar_func_op!(u32, min, self, rhs, N)
744    }
745    #[inline(always)]
746    unsafe fn max_u16(self, rhs: Self) -> Self {
747        scalar_func_op!(u16, max, self, rhs, N)
748    }
749    #[inline(always)]
750    unsafe fn min_u16(self, rhs: Self) -> Self {
751        scalar_func_op!(u16, min, self, rhs, N)
752    }
753    #[inline(always)]
754    unsafe fn max_u8(self, rhs: Self) -> Self {
755        scalar_func_op!(u8, max, self, rhs, N)
756    }
757    #[inline(always)]
758    unsafe fn min_u8(self, rhs: Self) -> Self {
759        scalar_func_op!(u8, min, self, rhs, N)
760    }
761}
762
763// TODO: Make a custom trait for handling this transmutation into i*.
764impl<const N: usize> SimdSplatImpl for ScalarReg<N> {
765    #[inline(always)]
766    unsafe fn splat_64<T>(val: T) -> Self {
767        scalar_splat!(u64, self, transmute_copy(&val), N)
768    }
769    #[inline(always)]
770    unsafe fn splat_32<T>(val: T) -> Self {
771        scalar_splat!(u32, self, transmute_copy(&val), N)
772    }
773    #[inline(always)]
774    unsafe fn splat_16<T>(val: T) -> Self {
775        scalar_splat!(u16, self, transmute_copy(&val), N)
776    }
777    #[inline(always)]
778    unsafe fn splat_8<T>(val: T) -> Self {
779        scalar_splat!(u8, self, transmute_copy(&val), N)
780    }
781}
782
783impl<const N: usize> SimdGatherImpl for ScalarReg<N> {
784    #[inline(always)]
785    unsafe fn gather_32_from_32<T, const B: i32>(self, ptr: *const T) -> Self {
786        unsafe {
787            let mut new = Self([0; N]);
788            let new_ptr = new.0.as_mut_ptr() as *mut u32;
789            let indices_ptr = self.0.as_ptr() as *const u32;
790            let data_ptr = ptr as *const u32;
791            for i in 0..(N / 4) {
792                let index = *indices_ptr.add(i) as usize;
793                *new_ptr.add(i) = *data_ptr.add(index);
794            }
795            new
796        }
797    }
798    #[inline(always)]
799    unsafe fn gather_64_from_64<T, const B: i32>(self, ptr: *const T) -> Self {
800        unsafe {
801            let mut new = Self([0; N]);
802            let new_ptr = new.0.as_mut_ptr() as *mut u64;
803            let indices_ptr = self.0.as_ptr() as *const u64;
804            let data_ptr = ptr as *const u64;
805            for i in 0..(N / 8) {
806                let index = *indices_ptr.add(i) as usize;
807                *new_ptr.add(i) = *data_ptr.add(index);
808            }
809            new
810        }
811    }
812}
813
814impl<const N: usize> SimdSqrtImpl for ScalarReg<N> {
815    #[inline(always)]
816    unsafe fn sqrt_f64(self) -> Self {
817        scalar_self_op!(f64, sqrt, self, N)
818    }
819    #[inline(always)]
820    unsafe fn sqrt_f32(self) -> Self {
821        scalar_self_op!(f32, sqrt, self, N)
822    }
823    #[inline(always)]
824    unsafe fn rsqrt_f32(self) -> Self {
825        unsafe { Self::splat_32(1.0f32).f32_div(self.sqrt_f32()) }
826    }
827}
828
829impl<const N: usize> SimdAllBitsImpl for ScalarMask<N> {
830    #[inline(always)]
831    unsafe fn all_zero(self) -> bool {
832        !self.0.iter().any(|&x| x)
833    }
834}
835
836impl<const N: usize> SimdBitwiseImpl for ScalarMask<N> {
837    #[inline(always)]
838    unsafe fn and(self, rhs: Self) -> Self {
839        Self(std::array::from_fn(|i| self.0[i] & rhs.0[i]))
840    }
841    #[inline(always)]
842    unsafe fn or(self, rhs: Self) -> Self {
843        Self(std::array::from_fn(|i| self.0[i] | rhs.0[i]))
844    }
845    #[inline(always)]
846    unsafe fn xor(self, rhs: Self) -> Self {
847        Self(std::array::from_fn(|i| self.0[i] ^ rhs.0[i]))
848    }
849    #[inline(always)]
850    unsafe fn not(self) -> Self {
851        Self(std::array::from_fn(|i| !self.0[i]))
852    }
853    #[inline(always)]
854    unsafe fn and_not(self, rhs: Self) -> Self {
855        unsafe { self.and(rhs.not()) }
856    }
857}
858
859impl<const N: usize> SimdNegateImpl for ScalarReg<N> {
860    #[inline(always)]
861    unsafe fn negate_f64(self) -> Self {
862        unsafe { Self::splat_64(-0.0f64).xor(self) }
863    }
864    #[inline(always)]
865    unsafe fn negate_f32(self) -> Self {
866        unsafe { Self::splat_32(-0.0f64).xor(self) }
867    }
868}
869
870impl<const N: usize> SimdBlockShiftImpl for ScalarReg<N> {
871    #[inline(always)]
872    unsafe fn block_left_byte_shift<const M: i32>(self) -> Self {
873        let mut new = unsafe { Self::splat_8(0) };
874        for block_start in (0..N).step_by(16) {
875            let block_end = block_start + 16;
876            for i in (block_start + M as usize)..block_end {
877                new.0[i] = self.0[i - M as usize];
878            }
879        }
880        new
881    }
882    #[inline(always)]
883    unsafe fn block_right_byte_shift<const M: i32>(self) -> Self {
884        let mut new = unsafe { Self::splat_8(0) };
885        for block_start in (0..N).step_by(16) {
886            let block_end = block_start + 16;
887            for i in block_start..(block_end - M as usize) {
888                new.0[i] = self.0[i + M as usize];
889            }
890        }
891        new
892    }
893}
894
895impl<const N: usize> SimdMaskBitConversion for ScalarMask<N> {
896    #[inline(always)]
897    unsafe fn to_bits_64(self) -> u64 {
898        let mut bits = 0u64;
899        for i in 0..(N >> 3) {
900            bits ^= (self.0[i] as u64) << i
901        }
902        bits
903    }
904    #[inline(always)]
905    unsafe fn to_bits_32(self) -> u64 {
906        let mut bits = 0u64;
907        for i in 0..(N >> 2) {
908            bits ^= (self.0[i] as u64) << i
909        }
910        bits
911    }
912    #[inline(always)]
913    unsafe fn to_bits_8(self) -> u64 {
914        let mut bits = 0u64;
915        for i in 0..N {
916            bits ^= (self.0[i] as u64) << i
917        }
918        bits
919    }
920    #[inline(always)]
921    unsafe fn from_bits_64(bitmask: u64) -> Self {
922        let mut new_mask = Self([false; N]);
923        for i in 0..N {
924            new_mask.0[i] = ((bitmask >> i) & 1) == 1;
925        }
926        new_mask
927    }
928    unsafe fn from_bits_32(bitmask: u64) -> Self {
929        let mut new_mask = Self([false; N]);
930        for i in 0..N {
931            new_mask.0[i] = ((bitmask >> i) & 1) == 1;
932        }
933        new_mask
934    }
935    unsafe fn from_bits_16(bitmask: u64) -> Self {
936        let mut new_mask = Self([false; N]);
937        for i in 0..N {
938            new_mask.0[i] = ((bitmask >> i) & 1) == 1;
939        }
940        new_mask
941    }
942    unsafe fn from_bits_8(bitmask: u64) -> Self {
943        let mut new_mask = Self([false; N]);
944        for i in 0..N {
945            new_mask.0[i] = ((bitmask >> i) & 1) == 1;
946        }
947        new_mask
948    }
949}
950
951impl<const M: usize> SimdLaneShiftImpl for ScalarReg<M> {
952    #[inline(always)]
953    unsafe fn left_lane_shift_32(self, n: u32) -> Self {
954        let mut new = unsafe { Self::zero() };
955        if n as usize * 4 >= M {
956            new
957        } else {
958            let bytes = (n * 4) as usize;
959            for i in 0..(M - bytes) {
960                new.0[i] = self.0[i + bytes];
961            }
962            new
963        }
964    }
965
966    #[inline(always)]
967    unsafe fn right_lane_shift_32(self, n: u32) -> Self {
968        let mut new = unsafe { Self::zero() };
969        if n as usize * 4 >= M {
970            new
971        } else {
972            let bytes = (n * 4) as usize;
973            for i in bytes..M {
974                new.0[i] = self.0[i - bytes];
975            }
976            new
977        }
978    }
979}