Skip to main content

simply_simd/register/
element.rs

1use std::fmt;
2use std::marker::PhantomData;
3use std::ops::*;
4
5use num_traits::NumCast;
6
7use crate::SimdElement;
8use crate::architectures::interface::*;
9use crate::simd_array::SimdToArray;
10use crate::mask::Mask;
11use crate::register::Simd;
12use crate::simd_types::{B32, B64, BitSize, SimdType};
13
14impl<T: SimdElement, A: Arch> Simd<T, A> {
15    #[inline(always)]
16    pub(crate) fn new(data: A::Vec) -> Self {
17        Self {
18            data,
19            _marker: PhantomData,
20        }
21    }
22
23    #[inline(always)]
24    pub fn zero() -> Self {
25        unsafe { Self::new(A::Vec::zero()) }
26    }
27
28    #[inline(always)]
29    pub fn from_aligned_slice(slice: &[T]) -> Self {
30        let ptr = slice.as_ptr();
31        assert!(ptr.align_offset(Self::SIMD_WIDTH) == 0);
32        assert!(slice.len() >= Self::LANES);
33        unsafe { Self::new(A::Vec::load_aligned(ptr)) }
34    }
35
36    /// # Safety
37    /// Does not check if the slice goes out of bounds.
38    #[inline(always)]
39    pub unsafe fn from_aligned_slice_unchecked(slice: &[T]) -> Self {
40        let ptr = slice.as_ptr();
41        debug_assert!(ptr.align_offset(Self::SIMD_WIDTH) == 0);
42        debug_assert!(slice.len() >= Self::LANES);
43        unsafe { Self::new(A::Vec::load_aligned(ptr)) }
44    }
45
46    #[inline(always)]
47    pub fn from_slice(slice: &[T]) -> Self {
48        if slice.len() >= Self::LANES {
49            unsafe { Self::new(A::Vec::load_unaligned(slice.as_ptr())) }
50        } else {
51            let mut array = Self::zero().to_array();
52            for (arr, val) in array.iter_mut().zip(slice.iter()) {
53                *arr = *val;
54            }
55            unsafe { Self::from_slice_unchecked(array.as_slice()) }
56        }
57    }
58
59    /// # Safety
60    /// Requires allocated memory to be behind (left) of the slice.
61    /// Bounds are not checked.
62    /// Length of the slice must be less than or equal to the number of lanes.
63    #[inline(always)]
64    pub unsafe fn from_slice_partial(slice: &[T]) -> Self {
65        debug_assert!(slice.len() <= Self::LANES);
66        unsafe {
67            let offset = Self::LANES - slice.len();
68            let raw_ptr = slice.as_ptr().sub(offset);
69            let simd = Self::new(A::Vec::load_unaligned(raw_ptr));
70            simd.left_lane_shift(offset as u32)
71        }
72    }
73
74    /// # Safety
75    /// Does not check if the slice goes out of bounds.
76    #[inline(always)]
77    pub unsafe fn from_slice_unchecked(slice: &[T]) -> Self {
78        debug_assert!(slice.len() >= Self::LANES);
79        unsafe { Self::new(A::Vec::load_unaligned(slice.as_ptr())) }
80    }
81
82    #[inline(always)]
83    pub fn copy_to_aligned_slice(self, slice: &mut [T]) {
84        let ptr = slice.as_mut_ptr();
85        assert!(ptr.align_offset(Self::SIMD_WIDTH) == 0);
86        assert!(slice.len() >= Self::LANES);
87        unsafe { self.data.store_aligned(ptr) };
88    }
89
90    /// # Safety
91    /// Does not check if the slice goes out of bounds.
92    #[inline(always)]
93    pub unsafe fn copy_to_aligned_slice_unchecked(self, slice: &mut [T]) {
94        let ptr = slice.as_mut_ptr();
95        debug_assert!(ptr.align_offset(Self::SIMD_WIDTH) == 0);
96        debug_assert!(slice.len() >= Self::LANES);
97        unsafe { self.data.store_aligned(ptr) };
98    }
99
100    #[inline(always)]
101    pub fn copy_to_slice(self, slice: &mut [T]) {
102        if slice.len() >= Self::LANES {
103            let ptr = slice.as_mut_ptr();
104            unsafe { self.data.store_unaligned(ptr) };
105        } else {
106            // Scalar/tail case.
107            let array = self.to_array();
108            slice
109                .iter_mut()
110                .zip(array.iter())
111                .for_each(|(src, new)| *src = *new);
112        }
113    }
114
115    /// # Safety
116    /// Does not check if the slice goes out of bounds.
117    #[inline(always)]
118    pub unsafe fn copy_to_slice_unchecked(self, slice: &mut [T]) {
119        let ptr = slice.as_mut_ptr();
120        debug_assert!(slice.len() >= Self::LANES);
121        unsafe { self.data.store_unaligned(ptr) };
122    }
123
124    /// Converts the Simd register into an array.
125    ///
126    /// # Example
127    ///
128    /// TODO
129    /// use simply_simd::
130    #[inline(always)]
131    pub fn to_array(self) -> T::Array<A> {
132        let mut array = T::Array::<A>::from_fn(|_| T::from(0).unwrap());
133        self.copy_to_slice(array.as_mut_slice());
134        array
135    }
136
137    #[inline(always)]
138    pub fn iota(offset: T) -> Self {
139        let iota_array =
140            T::Array::<A>::from_fn(|i| <T as NumCast>::from(i).unwrap().safe_add(offset));
141        Self::from_slice(iota_array.as_slice())
142    }
143}
144
145impl<T: SimdElement, A: Arch> fmt::Debug for Simd<T, A> {
146    #[inline(always)]
147    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
148        let buf = self.to_array();
149        write!(f, "{:?}", buf)
150    }
151}
152
153// === Assign operations ===
154impl<T: SimdElement, A: Arch> AddAssign for Simd<T, A>
155where
156    Self: Add<Output = Self> + Copy,
157{
158    #[inline(always)]
159    fn add_assign(&mut self, rhs: Self) {
160        *self = *self + rhs;
161    }
162}
163
164impl<T: SimdElement, A: Arch> SubAssign for Simd<T, A>
165where
166    Self: Sub<Output = Self> + Copy,
167{
168    #[inline(always)]
169    fn sub_assign(&mut self, rhs: Self) {
170        *self = *self - rhs;
171    }
172}
173
174impl<T: SimdElement, A: Arch> MulAssign for Simd<T, A>
175where
176    Self: Mul<Output = Self> + Copy,
177{
178    #[inline(always)]
179    fn mul_assign(&mut self, rhs: Self) {
180        *self = *self * rhs;
181    }
182}
183
184impl<T: SimdElement, A: Arch> DivAssign for Simd<T, A>
185where
186    Self: Div<Output = Self> + Copy,
187{
188    #[inline(always)]
189    fn div_assign(&mut self, rhs: Self) {
190        *self = *self / rhs;
191    }
192}
193
194impl<T: SimdElement, A: Arch> RemAssign for Simd<T, A>
195where
196    Self: Rem<Output = Self> + Copy,
197{
198    #[inline(always)]
199    fn rem_assign(&mut self, rhs: Self) {
200        *self = *self % rhs;
201    }
202}
203
204impl<T: SimdElement, A: Arch> BitAndAssign for Simd<T, A>
205where
206    Self: BitAnd<Output = Self> + Copy,
207{
208    #[inline(always)]
209    fn bitand_assign(&mut self, rhs: Self) {
210        *self = *self & rhs;
211    }
212}
213
214impl<T: SimdElement, A: Arch> BitOrAssign for Simd<T, A>
215where
216    Self: BitOr<Output = Self> + Copy,
217{
218    #[inline(always)]
219    fn bitor_assign(&mut self, rhs: Self) {
220        *self = *self | rhs;
221    }
222}
223
224impl<T: SimdElement, A: Arch> BitXorAssign for Simd<T, A>
225where
226    Self: BitXor<Output = Self> + Copy,
227{
228    #[inline(always)]
229    fn bitxor_assign(&mut self, rhs: Self) {
230        *self = *self ^ rhs;
231    }
232}
233
234impl<T: SimdElement, A: Arch> Neg for Simd<T, A> {
235    type Output = Self;
236    #[inline(always)]
237    fn neg(self) -> Self {
238        Self::zero() - self
239    }
240}
241
242impl<T: SimdElement, A: Arch> Simd<T, A> {
243    #[inline(always)]
244    pub fn raw_cast<S: SimdElement>(self) -> Simd<S, A> {
245        Simd::new(self.data)
246    }
247}
248
249impl<T: SimdElement, A: Arch> Default for Simd<T, A> {
250    #[inline(always)]
251    fn default() -> Self {
252        Self::splat(<T as NumCast>::from(T::default()).unwrap())
253    }
254}
255
256impl<T: SimdElement, A: Arch> Add for Simd<T, A> {
257    type Output = Self;
258    #[inline(always)]
259    fn add(self, rhs: Self) -> Self {
260        unsafe {
261            Self::new(match T::TYPE {
262                SimdType::F64 => self.data.f64_add(rhs.data),
263                SimdType::F32 => self.data.f32_add(rhs.data),
264                SimdType::I64 => self.data.i64_add(rhs.data),
265                SimdType::I32 => self.data.i32_add(rhs.data),
266                SimdType::I16 => self.data.i16_add(rhs.data),
267                SimdType::I8 => self.data.i8_add(rhs.data),
268                SimdType::U64 => self.data.i64_add(rhs.data),
269                SimdType::U32 => self.data.i32_add(rhs.data),
270                SimdType::U16 => self.data.i16_add(rhs.data),
271                SimdType::U8 => self.data.i8_add(rhs.data),
272            })
273        }
274    }
275}
276
277impl<T: SimdElement, A: Arch> Sub for Simd<T, A> {
278    type Output = Self;
279    #[inline(always)]
280    fn sub(self, rhs: Self) -> Self {
281        unsafe {
282            Self::new(match T::TYPE {
283                SimdType::F64 => self.data.f64_sub(rhs.data),
284                SimdType::F32 => self.data.f32_sub(rhs.data),
285                SimdType::I64 => self.data.i64_sub(rhs.data),
286                SimdType::I32 => self.data.i32_sub(rhs.data),
287                SimdType::I16 => self.data.i16_sub(rhs.data),
288                SimdType::I8 => self.data.i8_sub(rhs.data),
289                SimdType::U64 => self.data.i64_sub(rhs.data),
290                SimdType::U32 => self.data.i32_sub(rhs.data),
291                SimdType::U16 => self.data.i16_sub(rhs.data),
292                SimdType::U8 => self.data.i8_sub(rhs.data),
293            })
294        }
295    }
296}
297
298
299impl<T: SimdElement, A: Arch> Simd<T, A> {
300    /// Clamps the values in a register between two bounds, inclusive.
301    ///
302    /// # Parameters:
303    /// - `lower_bound`: Minimum value after clamping
304    /// - `upper_bound`: Maximum value after clamping
305    #[inline(always)]
306    pub fn clamp(self, lower_bound: Self, upper_bound: Self) -> Self {
307        self.min(upper_bound).max(lower_bound)
308    }
309
310    /// Shifts bytes N to the left within 128-bit blocks.
311    #[inline(always)]
312    pub fn block_left_byte_shift<const N: i32>(self) -> Self {
313        unsafe { Self::new(self.data.block_left_byte_shift::<N>()) }
314    }
315
316    /// Shifts bytes N to the right within 128-bit blocks.
317    #[inline(always)]
318    pub fn block_right_byte_shift<const N: i32>(self) -> Self {
319        unsafe { Self::new(self.data.block_right_byte_shift::<N>()) }
320    }
321
322    /// Blends two registers using an immediate mask.
323    #[inline(always)]
324    pub fn blend<const N: i32>(self, false_values: Self) -> Self {
325        unsafe {
326            match T::BIT_SIZE {
327                BitSize::Size64 => Self::new(self.data.blend_32::<N>(false_values.data)),
328                BitSize::Size32 => Self::new(self.data.blend_32::<N>(false_values.data)),
329                _ => unreachable!(), // TODO: Add 16 and 8 for immediate blend.
330            }
331        }
332    }
333}
334
335impl<T: SimdElement, A: Arch> Simd<T, A> {
336    /// Broadcasts a value across the entire register.
337    #[inline(always)]
338    pub fn splat(val: T) -> Self {
339        unsafe {
340            Self::new(match T::BIT_SIZE {
341                BitSize::Size64 => A::Vec::splat_64(val),
342                BitSize::Size32 => A::Vec::splat_32(val),
343                BitSize::Size16 => A::Vec::splat_16(val),
344                BitSize::Size8 => A::Vec::splat_8(val),
345            })
346        }
347    }
348
349    /// Loads a register according to a mask.
350    #[inline(always)]
351    pub fn masked_load(slice: &[T], mask: Mask<T, A>) -> Self {
352        unsafe {
353            Self::new(match T::BIT_SIZE {
354                BitSize::Size64 => A::Vec::masked_load_64(slice.as_ptr(), mask.data),
355                BitSize::Size32 => A::Vec::masked_load_32(slice.as_ptr(), mask.data),
356                _ => unreachable!(),
357            })
358        }
359    }
360
361    /// Loads only the first `amount` elements into the register.
362    pub fn partial_load(slice: &[T], amount: usize) -> Self {
363        assert!(slice.len() >= Self::LANES, "Attempted to do a partial load, but index is out of bounds!");
364        let amount_vec = Self::splat(<T as NumCast>::from(amount).unwrap());
365        let mask = Self::iota(<T as NumCast>::from(0).unwrap()).simd_lt(amount_vec);
366        Self::masked_load(slice, mask)
367    }
368
369    /// Loads only the first `amount` elements into the register.
370    ///
371    /// # Safety
372    /// - Slice must be greater than or equal to `ArchSimd::<T>::LANES`
373    pub unsafe fn partial_load_unchecked(slice: &[T], amount: usize) -> Self {
374        debug_assert!(slice.len() >= Self::LANES, "Index is out of bounds in unsafe code!");
375        let amount_vec = Self::splat(<T as NumCast>::from(amount).unwrap());
376        let mask = Self::iota(<T as NumCast>::from(0).unwrap()).simd_lt(amount_vec);
377        Self::masked_load(slice, mask)
378    }
379
380    /// Stores the register using a given mask.
381    #[inline(always)]
382    pub fn masked_store(self, slice: &mut [T], mask: Mask<T, A>) {
383        unsafe {
384            match T::BIT_SIZE {
385                BitSize::Size64 => {
386                    A::Vec::masked_store_64(self.data, slice.as_mut_ptr(), mask.data)
387                }
388                BitSize::Size32 => {
389                    A::Vec::masked_store_32(self.data, slice.as_mut_ptr(), mask.data)
390                }
391                _ => unreachable!(), // TODO: ADD SUPPORT AOR OTHER SIZES!!!
392            }
393        }
394    }
395
396    /// Stores only the first `amount` elements into the register.
397    pub fn partial_store(self, slice: &mut [T], amount: usize) {
398        match T::BIT_SIZE {
399            BitSize::Size64 => {
400                let iota = Simd::iota(0u64);
401                let n_vec = Simd::splat(amount as u64);
402                let mask = n_vec.simd_gt(iota);
403                Self::masked_store(self, slice, mask.raw_cast());
404            }
405            BitSize::Size32 => {
406                let iota = Simd::iota(0u32);
407                let n_vec = Simd::splat(amount as u32);
408                let mask = n_vec.simd_gt(iota);
409                Self::masked_store(self, slice, mask.raw_cast());
410            }
411            _ => unreachable!(),
412        }
413    }
414
415    #[inline(always)]
416    pub fn simd_eq(self, rhs: Self) -> Mask<T, A> {
417        unsafe {
418            Mask::new(match T::TYPE {
419                SimdType::F64 => self.data.cmp_f64_eq(rhs.data),
420                SimdType::F32 => self.data.cmp_f32_eq(rhs.data),
421                SimdType::U64 => self.data.cmp_i64_eq(rhs.data),
422                SimdType::U32 => self.data.cmp_i32_eq(rhs.data),
423                SimdType::U16 => self.data.cmp_i16_eq(rhs.data),
424                SimdType::U8 => self.data.cmp_i8_eq(rhs.data),
425                SimdType::I64 => self.data.cmp_i64_eq(rhs.data),
426                SimdType::I32 => self.data.cmp_i32_eq(rhs.data),
427                SimdType::I16 => self.data.cmp_i16_eq(rhs.data),
428                SimdType::I8 => self.data.cmp_i8_eq(rhs.data),
429                // _ => unreachable!() // TODO: Add integer types .
430            })
431        }
432    }
433
434    #[inline(always)]
435    pub fn simd_ne(self, rhs: Self) -> Mask<T, A> {
436        unsafe {
437            Mask::new(match T::TYPE {
438                SimdType::F64 => self.data.cmp_f64_neq(rhs.data),
439                SimdType::F32 => self.data.cmp_f32_neq(rhs.data),
440                _ => self.simd_eq(rhs).data,
441            })
442        }
443    }
444
445    #[inline(always)]
446    pub fn simd_gt(self, rhs: Self) -> Mask<T, A> {
447        unsafe {
448            Mask::new(match T::TYPE {
449                SimdType::F64 => self.data.cmp_f64_gt(rhs.data),
450                SimdType::F32 => self.data.cmp_f32_gt(rhs.data),
451                SimdType::U64 => self.data.cmp_i64_gt(rhs.data),
452                SimdType::U32 => self.data.cmp_i32_gt(rhs.data),
453                SimdType::U16 => self.data.cmp_i16_gt(rhs.data),
454                SimdType::U8 => self.data.cmp_i8_gt(rhs.data),
455                SimdType::I64 => self.data.cmp_i64_gt(rhs.data),
456                SimdType::I32 => self.data.cmp_i32_gt(rhs.data),
457                SimdType::I16 => self.data.cmp_i16_gt(rhs.data),
458                SimdType::I8 => self.data.cmp_i8_gt(rhs.data),
459                // _ => unreachable!() // TODO: Add integer types .
460            })
461        }
462    }
463
464    // TODO: Aind better way to handle comparisons.
465    #[inline(always)]
466    pub fn simd_ge(self, rhs: Self) -> Mask<T, A> {
467        unsafe {
468            Mask::new(match T::TYPE {
469                SimdType::F64 => self.data.cmp_f64_ge(rhs.data),
470                SimdType::F32 => self.data.cmp_f32_ge(rhs.data),
471                // SimdType::U64 => self.data.cmp_i64_gt(rhs.data).or(self.data.cmp_i64_eq(rhs.data)),
472                // SimdType::U32 => self.data.cmp_i32_gt(rhs.data).or(self.data.cmp_i32_eq(rhs.data)),
473                // SimdType::U16 => self.data.cmp_i16_gt(rhs.data).or(self.data.cmp_i16_eq(rhs.data)),
474                // SimdType::U8 => self.data.cmp_i8_gt(rhs.data).or(self.data.cmp_i8_eq(rhs.data)),
475                SimdType::I64 => self
476                    .data
477                    .cmp_i64_gt(rhs.data)
478                    .or(self.data.cmp_i64_eq(rhs.data)),
479                SimdType::I32 => self
480                    .data
481                    .cmp_i32_gt(rhs.data)
482                    .or(self.data.cmp_i32_eq(rhs.data)),
483                SimdType::I16 => self
484                    .data
485                    .cmp_i16_gt(rhs.data)
486                    .or(self.data.cmp_i16_eq(rhs.data)),
487                SimdType::I8 => self
488                    .data
489                    .cmp_i8_gt(rhs.data)
490                    .or(self.data.cmp_i8_eq(rhs.data)),
491                _ => panic!("Unsigned integer types for less than not implemented!"), // TODO: Add integer types .
492            })
493        }
494    }
495
496    #[inline(always)]
497    pub fn simd_lt(self, rhs: Self) -> Mask<T, A> {
498        unsafe {
499            Mask::new(match T::TYPE {
500                SimdType::F64 => self.data.cmp_f64_lt(rhs.data),
501                SimdType::F32 => self.data.cmp_f32_lt(rhs.data),
502                _ => panic!("Less than for integers not implemented!"), // TODO: Add integer types .
503            })
504        }
505    }
506
507    #[inline(always)]
508    pub fn simd_le(self, rhs: Self) -> Mask<T, A> {
509        unsafe {
510            Mask::new(match T::TYPE {
511                SimdType::F64 => self.data.cmp_f64_le(rhs.data),
512                SimdType::F32 => self.data.cmp_f32_le(rhs.data),
513                _ => panic!("Less than or equal not implemented for integers!"), // TODO: Add integer types .
514            })
515        }
516    }
517
518    // TODO: Handle max for U64/I64.
519    pub fn max(self, rhs: Self) -> Self {
520        unsafe {
521            Self::new(match T::TYPE {
522                SimdType::F64 => self.data.max_f64(rhs.data),
523                SimdType::F32 => self.data.max_f32(rhs.data),
524                SimdType::I32 => self.data.max_i32(rhs.data),
525                SimdType::I16 => self.data.max_i16(rhs.data),
526                SimdType::I8 => self.data.max_i8(rhs.data),
527                SimdType::U32 => self.data.max_u32(rhs.data),
528                SimdType::U16 => self.data.max_u16(rhs.data),
529                SimdType::U8 => self.data.max_u8(rhs.data),
530                _ => panic!("Max for U64/I64 not implemented!"),
531            })
532        }
533    }
534
535    pub fn min(self, rhs: Self) -> Self {
536        unsafe {
537            Self::new(match T::TYPE {
538                SimdType::F64 => self.data.min_f64(rhs.data),
539                SimdType::F32 => self.data.min_f32(rhs.data),
540                SimdType::I32 => self.data.min_i32(rhs.data),
541                SimdType::I16 => self.data.min_i16(rhs.data),
542                SimdType::I8 => self.data.min_i8(rhs.data),
543                SimdType::U32 => self.data.min_u32(rhs.data),
544                SimdType::U16 => self.data.min_u16(rhs.data),
545                SimdType::U8 => self.data.min_u8(rhs.data),
546                _ => panic!("Min for U64/I64 not implemented!"),
547            })
548        }
549    }
550}
551
552impl<T: SimdElement + SimdElement<BitWidthType = B32>, A: Arch> Simd<T, A> {
553    #[inline(always)]
554    pub fn permute_32(self, indices: Simd<u32, A>) -> Self {
555        unsafe { Self::new(self.data.permute_32(indices.data)) }
556    }
557}
558
559impl<T: SimdElement, A: Arch> Simd<T, A> {
560    #[inline(always)]
561    pub fn permute_8(self, indices: Simd<u8, A>) -> Self {
562        unsafe { Self::new(self.data.permute_8(indices.data)) }
563    }
564
565    #[inline(always)]
566    pub fn permute_8_pattern_32(self, indices: [u8; 4]) -> Self {
567        let pattern = u32::from_ne_bytes(indices);
568        let pattern_vec = Simd::<u32, A>::splat(pattern);
569        unsafe { Self::new(self.data.permute_8(pattern_vec.data)) }
570    }
571}
572
573// TODO: Super early version gather.
574impl<A: Arch> Simd<u32, A> {
575    pub fn gather<S: SimdElement + SimdElement<BitWidthType = B32>, const N: usize>(
576        self,
577        slice: &[S; N],
578    ) -> Simd<S, A> {
579        if N <= Self::LANES {
580            let data = Simd::<S, A>::from_slice(&slice[..]);
581            data.permute_32(self)
582        } else {
583            unsafe { Simd::new(self.data.gather_32_from_32::<S, 4>(slice.as_ptr())) }
584        }
585    }
586}
587
588impl<A: Arch> Simd<u64, A> {
589    pub fn gather<S: SimdElement + SimdElement<BitWidthType = B64>, const N: usize>(
590        self,
591        slice: &[S; N],
592    ) -> Simd<S, A> {
593        unsafe { Simd::new(self.data.gather_64_from_64::<S, 8>(slice.as_ptr())) }
594    }
595}
596
597// TODO: Add other types of lane shifts.
598impl<T: SimdElement, A: Arch> Simd<T, A> {
599    pub fn left_lane_shift(self, n: u32) -> Self {
600        match T::BIT_SIZE {
601            BitSize::Size32 => unsafe { Self::new(self.data.left_lane_shift_32(n)) },
602            _ => unreachable!(),
603        }
604    }
605    pub fn right_lane_shift(self, n: u32) -> Self {
606        match T::BIT_SIZE {
607            BitSize::Size32 => unsafe { Self::new(self.data.right_lane_shift_32(n)) },
608            _ => unreachable!(),
609        }
610    }
611}