1use std::fmt;
2use std::marker::PhantomData;
3use std::ops::*;
4
5use num_traits::NumCast;
6
7use crate::SimdElement;
8use crate::architectures::interface::*;
9use crate::simd_array::SimdToArray;
10use crate::mask::Mask;
11use crate::register::Simd;
12use crate::simd_types::{B32, B64, BitSize, SimdType};
13
14impl<T: SimdElement, A: Arch> Simd<T, A> {
15 #[inline(always)]
16 pub(crate) fn new(data: A::Vec) -> Self {
17 Self {
18 data,
19 _marker: PhantomData,
20 }
21 }
22
23 #[inline(always)]
24 pub fn zero() -> Self {
25 unsafe { Self::new(A::Vec::zero()) }
26 }
27
28 #[inline(always)]
29 pub fn from_aligned_slice(slice: &[T]) -> Self {
30 let ptr = slice.as_ptr();
31 assert!(ptr.align_offset(Self::SIMD_WIDTH) == 0);
32 assert!(slice.len() >= Self::LANES);
33 unsafe { Self::new(A::Vec::load_aligned(ptr)) }
34 }
35
36 #[inline(always)]
39 pub unsafe fn from_aligned_slice_unchecked(slice: &[T]) -> Self {
40 let ptr = slice.as_ptr();
41 debug_assert!(ptr.align_offset(Self::SIMD_WIDTH) == 0);
42 debug_assert!(slice.len() >= Self::LANES);
43 unsafe { Self::new(A::Vec::load_aligned(ptr)) }
44 }
45
46 #[inline(always)]
47 pub fn from_slice(slice: &[T]) -> Self {
48 if slice.len() >= Self::LANES {
49 unsafe { Self::new(A::Vec::load_unaligned(slice.as_ptr())) }
50 } else {
51 let mut array = Self::zero().to_array();
52 for (arr, val) in array.iter_mut().zip(slice.iter()) {
53 *arr = *val;
54 }
55 unsafe { Self::from_slice_unchecked(array.as_slice()) }
56 }
57 }
58
59 #[inline(always)]
64 pub unsafe fn from_slice_partial(slice: &[T]) -> Self {
65 debug_assert!(slice.len() <= Self::LANES);
66 unsafe {
67 let offset = Self::LANES - slice.len();
68 let raw_ptr = slice.as_ptr().sub(offset);
69 let simd = Self::new(A::Vec::load_unaligned(raw_ptr));
70 simd.left_lane_shift(offset as u32)
71 }
72 }
73
74 #[inline(always)]
77 pub unsafe fn from_slice_unchecked(slice: &[T]) -> Self {
78 debug_assert!(slice.len() >= Self::LANES);
79 unsafe { Self::new(A::Vec::load_unaligned(slice.as_ptr())) }
80 }
81
82 #[inline(always)]
83 pub fn copy_to_aligned_slice(self, slice: &mut [T]) {
84 let ptr = slice.as_mut_ptr();
85 assert!(ptr.align_offset(Self::SIMD_WIDTH) == 0);
86 assert!(slice.len() >= Self::LANES);
87 unsafe { self.data.store_aligned(ptr) };
88 }
89
90 #[inline(always)]
93 pub unsafe fn copy_to_aligned_slice_unchecked(self, slice: &mut [T]) {
94 let ptr = slice.as_mut_ptr();
95 debug_assert!(ptr.align_offset(Self::SIMD_WIDTH) == 0);
96 debug_assert!(slice.len() >= Self::LANES);
97 unsafe { self.data.store_aligned(ptr) };
98 }
99
100 #[inline(always)]
101 pub fn copy_to_slice(self, slice: &mut [T]) {
102 if slice.len() >= Self::LANES {
103 let ptr = slice.as_mut_ptr();
104 unsafe { self.data.store_unaligned(ptr) };
105 } else {
106 let array = self.to_array();
108 slice
109 .iter_mut()
110 .zip(array.iter())
111 .for_each(|(src, new)| *src = *new);
112 }
113 }
114
115 #[inline(always)]
118 pub unsafe fn copy_to_slice_unchecked(self, slice: &mut [T]) {
119 let ptr = slice.as_mut_ptr();
120 debug_assert!(slice.len() >= Self::LANES);
121 unsafe { self.data.store_unaligned(ptr) };
122 }
123
124 #[inline(always)]
131 pub fn to_array(self) -> T::Array<A> {
132 let mut array = T::Array::<A>::from_fn(|_| T::from(0).unwrap());
133 self.copy_to_slice(array.as_mut_slice());
134 array
135 }
136
137 #[inline(always)]
138 pub fn iota(offset: T) -> Self {
139 let iota_array =
140 T::Array::<A>::from_fn(|i| <T as NumCast>::from(i).unwrap().safe_add(offset));
141 Self::from_slice(iota_array.as_slice())
142 }
143}
144
145impl<T: SimdElement, A: Arch> fmt::Debug for Simd<T, A> {
146 #[inline(always)]
147 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
148 let buf = self.to_array();
149 write!(f, "{:?}", buf)
150 }
151}
152
153impl<T: SimdElement, A: Arch> AddAssign for Simd<T, A>
155where
156 Self: Add<Output = Self> + Copy,
157{
158 #[inline(always)]
159 fn add_assign(&mut self, rhs: Self) {
160 *self = *self + rhs;
161 }
162}
163
164impl<T: SimdElement, A: Arch> SubAssign for Simd<T, A>
165where
166 Self: Sub<Output = Self> + Copy,
167{
168 #[inline(always)]
169 fn sub_assign(&mut self, rhs: Self) {
170 *self = *self - rhs;
171 }
172}
173
174impl<T: SimdElement, A: Arch> MulAssign for Simd<T, A>
175where
176 Self: Mul<Output = Self> + Copy,
177{
178 #[inline(always)]
179 fn mul_assign(&mut self, rhs: Self) {
180 *self = *self * rhs;
181 }
182}
183
184impl<T: SimdElement, A: Arch> DivAssign for Simd<T, A>
185where
186 Self: Div<Output = Self> + Copy,
187{
188 #[inline(always)]
189 fn div_assign(&mut self, rhs: Self) {
190 *self = *self / rhs;
191 }
192}
193
194impl<T: SimdElement, A: Arch> RemAssign for Simd<T, A>
195where
196 Self: Rem<Output = Self> + Copy,
197{
198 #[inline(always)]
199 fn rem_assign(&mut self, rhs: Self) {
200 *self = *self % rhs;
201 }
202}
203
204impl<T: SimdElement, A: Arch> BitAndAssign for Simd<T, A>
205where
206 Self: BitAnd<Output = Self> + Copy,
207{
208 #[inline(always)]
209 fn bitand_assign(&mut self, rhs: Self) {
210 *self = *self & rhs;
211 }
212}
213
214impl<T: SimdElement, A: Arch> BitOrAssign for Simd<T, A>
215where
216 Self: BitOr<Output = Self> + Copy,
217{
218 #[inline(always)]
219 fn bitor_assign(&mut self, rhs: Self) {
220 *self = *self | rhs;
221 }
222}
223
224impl<T: SimdElement, A: Arch> BitXorAssign for Simd<T, A>
225where
226 Self: BitXor<Output = Self> + Copy,
227{
228 #[inline(always)]
229 fn bitxor_assign(&mut self, rhs: Self) {
230 *self = *self ^ rhs;
231 }
232}
233
234impl<T: SimdElement, A: Arch> Neg for Simd<T, A> {
235 type Output = Self;
236 #[inline(always)]
237 fn neg(self) -> Self {
238 Self::zero() - self
239 }
240}
241
242impl<T: SimdElement, A: Arch> Simd<T, A> {
243 #[inline(always)]
244 pub fn raw_cast<S: SimdElement>(self) -> Simd<S, A> {
245 Simd::new(self.data)
246 }
247}
248
249impl<T: SimdElement, A: Arch> Default for Simd<T, A> {
250 #[inline(always)]
251 fn default() -> Self {
252 Self::splat(<T as NumCast>::from(T::default()).unwrap())
253 }
254}
255
256impl<T: SimdElement, A: Arch> Add for Simd<T, A> {
257 type Output = Self;
258 #[inline(always)]
259 fn add(self, rhs: Self) -> Self {
260 unsafe {
261 Self::new(match T::TYPE {
262 SimdType::F64 => self.data.f64_add(rhs.data),
263 SimdType::F32 => self.data.f32_add(rhs.data),
264 SimdType::I64 => self.data.i64_add(rhs.data),
265 SimdType::I32 => self.data.i32_add(rhs.data),
266 SimdType::I16 => self.data.i16_add(rhs.data),
267 SimdType::I8 => self.data.i8_add(rhs.data),
268 SimdType::U64 => self.data.i64_add(rhs.data),
269 SimdType::U32 => self.data.i32_add(rhs.data),
270 SimdType::U16 => self.data.i16_add(rhs.data),
271 SimdType::U8 => self.data.i8_add(rhs.data),
272 })
273 }
274 }
275}
276
277impl<T: SimdElement, A: Arch> Sub for Simd<T, A> {
278 type Output = Self;
279 #[inline(always)]
280 fn sub(self, rhs: Self) -> Self {
281 unsafe {
282 Self::new(match T::TYPE {
283 SimdType::F64 => self.data.f64_sub(rhs.data),
284 SimdType::F32 => self.data.f32_sub(rhs.data),
285 SimdType::I64 => self.data.i64_sub(rhs.data),
286 SimdType::I32 => self.data.i32_sub(rhs.data),
287 SimdType::I16 => self.data.i16_sub(rhs.data),
288 SimdType::I8 => self.data.i8_sub(rhs.data),
289 SimdType::U64 => self.data.i64_sub(rhs.data),
290 SimdType::U32 => self.data.i32_sub(rhs.data),
291 SimdType::U16 => self.data.i16_sub(rhs.data),
292 SimdType::U8 => self.data.i8_sub(rhs.data),
293 })
294 }
295 }
296}
297
298
299impl<T: SimdElement, A: Arch> Simd<T, A> {
300 #[inline(always)]
306 pub fn clamp(self, lower_bound: Self, upper_bound: Self) -> Self {
307 self.min(upper_bound).max(lower_bound)
308 }
309
310 #[inline(always)]
312 pub fn block_left_byte_shift<const N: i32>(self) -> Self {
313 unsafe { Self::new(self.data.block_left_byte_shift::<N>()) }
314 }
315
316 #[inline(always)]
318 pub fn block_right_byte_shift<const N: i32>(self) -> Self {
319 unsafe { Self::new(self.data.block_right_byte_shift::<N>()) }
320 }
321
322 #[inline(always)]
324 pub fn blend<const N: i32>(self, false_values: Self) -> Self {
325 unsafe {
326 match T::BIT_SIZE {
327 BitSize::Size64 => Self::new(self.data.blend_32::<N>(false_values.data)),
328 BitSize::Size32 => Self::new(self.data.blend_32::<N>(false_values.data)),
329 _ => unreachable!(), }
331 }
332 }
333}
334
335impl<T: SimdElement, A: Arch> Simd<T, A> {
336 #[inline(always)]
338 pub fn splat(val: T) -> Self {
339 unsafe {
340 Self::new(match T::BIT_SIZE {
341 BitSize::Size64 => A::Vec::splat_64(val),
342 BitSize::Size32 => A::Vec::splat_32(val),
343 BitSize::Size16 => A::Vec::splat_16(val),
344 BitSize::Size8 => A::Vec::splat_8(val),
345 })
346 }
347 }
348
349 #[inline(always)]
351 pub fn masked_load(slice: &[T], mask: Mask<T, A>) -> Self {
352 unsafe {
353 Self::new(match T::BIT_SIZE {
354 BitSize::Size64 => A::Vec::masked_load_64(slice.as_ptr(), mask.data),
355 BitSize::Size32 => A::Vec::masked_load_32(slice.as_ptr(), mask.data),
356 _ => unreachable!(),
357 })
358 }
359 }
360
361 pub fn partial_load(slice: &[T], amount: usize) -> Self {
363 assert!(slice.len() >= Self::LANES, "Attempted to do a partial load, but index is out of bounds!");
364 let amount_vec = Self::splat(<T as NumCast>::from(amount).unwrap());
365 let mask = Self::iota(<T as NumCast>::from(0).unwrap()).simd_lt(amount_vec);
366 Self::masked_load(slice, mask)
367 }
368
369 pub unsafe fn partial_load_unchecked(slice: &[T], amount: usize) -> Self {
374 debug_assert!(slice.len() >= Self::LANES, "Index is out of bounds in unsafe code!");
375 let amount_vec = Self::splat(<T as NumCast>::from(amount).unwrap());
376 let mask = Self::iota(<T as NumCast>::from(0).unwrap()).simd_lt(amount_vec);
377 Self::masked_load(slice, mask)
378 }
379
380 #[inline(always)]
382 pub fn masked_store(self, slice: &mut [T], mask: Mask<T, A>) {
383 unsafe {
384 match T::BIT_SIZE {
385 BitSize::Size64 => {
386 A::Vec::masked_store_64(self.data, slice.as_mut_ptr(), mask.data)
387 }
388 BitSize::Size32 => {
389 A::Vec::masked_store_32(self.data, slice.as_mut_ptr(), mask.data)
390 }
391 _ => unreachable!(), }
393 }
394 }
395
396 pub fn partial_store(self, slice: &mut [T], amount: usize) {
398 match T::BIT_SIZE {
399 BitSize::Size64 => {
400 let iota = Simd::iota(0u64);
401 let n_vec = Simd::splat(amount as u64);
402 let mask = n_vec.simd_gt(iota);
403 Self::masked_store(self, slice, mask.raw_cast());
404 }
405 BitSize::Size32 => {
406 let iota = Simd::iota(0u32);
407 let n_vec = Simd::splat(amount as u32);
408 let mask = n_vec.simd_gt(iota);
409 Self::masked_store(self, slice, mask.raw_cast());
410 }
411 _ => unreachable!(),
412 }
413 }
414
415 #[inline(always)]
416 pub fn simd_eq(self, rhs: Self) -> Mask<T, A> {
417 unsafe {
418 Mask::new(match T::TYPE {
419 SimdType::F64 => self.data.cmp_f64_eq(rhs.data),
420 SimdType::F32 => self.data.cmp_f32_eq(rhs.data),
421 SimdType::U64 => self.data.cmp_i64_eq(rhs.data),
422 SimdType::U32 => self.data.cmp_i32_eq(rhs.data),
423 SimdType::U16 => self.data.cmp_i16_eq(rhs.data),
424 SimdType::U8 => self.data.cmp_i8_eq(rhs.data),
425 SimdType::I64 => self.data.cmp_i64_eq(rhs.data),
426 SimdType::I32 => self.data.cmp_i32_eq(rhs.data),
427 SimdType::I16 => self.data.cmp_i16_eq(rhs.data),
428 SimdType::I8 => self.data.cmp_i8_eq(rhs.data),
429 })
431 }
432 }
433
434 #[inline(always)]
435 pub fn simd_ne(self, rhs: Self) -> Mask<T, A> {
436 unsafe {
437 Mask::new(match T::TYPE {
438 SimdType::F64 => self.data.cmp_f64_neq(rhs.data),
439 SimdType::F32 => self.data.cmp_f32_neq(rhs.data),
440 _ => self.simd_eq(rhs).data,
441 })
442 }
443 }
444
445 #[inline(always)]
446 pub fn simd_gt(self, rhs: Self) -> Mask<T, A> {
447 unsafe {
448 Mask::new(match T::TYPE {
449 SimdType::F64 => self.data.cmp_f64_gt(rhs.data),
450 SimdType::F32 => self.data.cmp_f32_gt(rhs.data),
451 SimdType::U64 => self.data.cmp_i64_gt(rhs.data),
452 SimdType::U32 => self.data.cmp_i32_gt(rhs.data),
453 SimdType::U16 => self.data.cmp_i16_gt(rhs.data),
454 SimdType::U8 => self.data.cmp_i8_gt(rhs.data),
455 SimdType::I64 => self.data.cmp_i64_gt(rhs.data),
456 SimdType::I32 => self.data.cmp_i32_gt(rhs.data),
457 SimdType::I16 => self.data.cmp_i16_gt(rhs.data),
458 SimdType::I8 => self.data.cmp_i8_gt(rhs.data),
459 })
461 }
462 }
463
464 #[inline(always)]
466 pub fn simd_ge(self, rhs: Self) -> Mask<T, A> {
467 unsafe {
468 Mask::new(match T::TYPE {
469 SimdType::F64 => self.data.cmp_f64_ge(rhs.data),
470 SimdType::F32 => self.data.cmp_f32_ge(rhs.data),
471 SimdType::I64 => self
476 .data
477 .cmp_i64_gt(rhs.data)
478 .or(self.data.cmp_i64_eq(rhs.data)),
479 SimdType::I32 => self
480 .data
481 .cmp_i32_gt(rhs.data)
482 .or(self.data.cmp_i32_eq(rhs.data)),
483 SimdType::I16 => self
484 .data
485 .cmp_i16_gt(rhs.data)
486 .or(self.data.cmp_i16_eq(rhs.data)),
487 SimdType::I8 => self
488 .data
489 .cmp_i8_gt(rhs.data)
490 .or(self.data.cmp_i8_eq(rhs.data)),
491 _ => panic!("Unsigned integer types for less than not implemented!"), })
493 }
494 }
495
496 #[inline(always)]
497 pub fn simd_lt(self, rhs: Self) -> Mask<T, A> {
498 unsafe {
499 Mask::new(match T::TYPE {
500 SimdType::F64 => self.data.cmp_f64_lt(rhs.data),
501 SimdType::F32 => self.data.cmp_f32_lt(rhs.data),
502 _ => panic!("Less than for integers not implemented!"), })
504 }
505 }
506
507 #[inline(always)]
508 pub fn simd_le(self, rhs: Self) -> Mask<T, A> {
509 unsafe {
510 Mask::new(match T::TYPE {
511 SimdType::F64 => self.data.cmp_f64_le(rhs.data),
512 SimdType::F32 => self.data.cmp_f32_le(rhs.data),
513 _ => panic!("Less than or equal not implemented for integers!"), })
515 }
516 }
517
518 pub fn max(self, rhs: Self) -> Self {
520 unsafe {
521 Self::new(match T::TYPE {
522 SimdType::F64 => self.data.max_f64(rhs.data),
523 SimdType::F32 => self.data.max_f32(rhs.data),
524 SimdType::I32 => self.data.max_i32(rhs.data),
525 SimdType::I16 => self.data.max_i16(rhs.data),
526 SimdType::I8 => self.data.max_i8(rhs.data),
527 SimdType::U32 => self.data.max_u32(rhs.data),
528 SimdType::U16 => self.data.max_u16(rhs.data),
529 SimdType::U8 => self.data.max_u8(rhs.data),
530 _ => panic!("Max for U64/I64 not implemented!"),
531 })
532 }
533 }
534
535 pub fn min(self, rhs: Self) -> Self {
536 unsafe {
537 Self::new(match T::TYPE {
538 SimdType::F64 => self.data.min_f64(rhs.data),
539 SimdType::F32 => self.data.min_f32(rhs.data),
540 SimdType::I32 => self.data.min_i32(rhs.data),
541 SimdType::I16 => self.data.min_i16(rhs.data),
542 SimdType::I8 => self.data.min_i8(rhs.data),
543 SimdType::U32 => self.data.min_u32(rhs.data),
544 SimdType::U16 => self.data.min_u16(rhs.data),
545 SimdType::U8 => self.data.min_u8(rhs.data),
546 _ => panic!("Min for U64/I64 not implemented!"),
547 })
548 }
549 }
550}
551
552impl<T: SimdElement + SimdElement<BitWidthType = B32>, A: Arch> Simd<T, A> {
553 #[inline(always)]
554 pub fn permute_32(self, indices: Simd<u32, A>) -> Self {
555 unsafe { Self::new(self.data.permute_32(indices.data)) }
556 }
557}
558
559impl<T: SimdElement, A: Arch> Simd<T, A> {
560 #[inline(always)]
561 pub fn permute_8(self, indices: Simd<u8, A>) -> Self {
562 unsafe { Self::new(self.data.permute_8(indices.data)) }
563 }
564
565 #[inline(always)]
566 pub fn permute_8_pattern_32(self, indices: [u8; 4]) -> Self {
567 let pattern = u32::from_ne_bytes(indices);
568 let pattern_vec = Simd::<u32, A>::splat(pattern);
569 unsafe { Self::new(self.data.permute_8(pattern_vec.data)) }
570 }
571}
572
573impl<A: Arch> Simd<u32, A> {
575 pub fn gather<S: SimdElement + SimdElement<BitWidthType = B32>, const N: usize>(
576 self,
577 slice: &[S; N],
578 ) -> Simd<S, A> {
579 if N <= Self::LANES {
580 let data = Simd::<S, A>::from_slice(&slice[..]);
581 data.permute_32(self)
582 } else {
583 unsafe { Simd::new(self.data.gather_32_from_32::<S, 4>(slice.as_ptr())) }
584 }
585 }
586}
587
588impl<A: Arch> Simd<u64, A> {
589 pub fn gather<S: SimdElement + SimdElement<BitWidthType = B64>, const N: usize>(
590 self,
591 slice: &[S; N],
592 ) -> Simd<S, A> {
593 unsafe { Simd::new(self.data.gather_64_from_64::<S, 8>(slice.as_ptr())) }
594 }
595}
596
597impl<T: SimdElement, A: Arch> Simd<T, A> {
599 pub fn left_lane_shift(self, n: u32) -> Self {
600 match T::BIT_SIZE {
601 BitSize::Size32 => unsafe { Self::new(self.data.left_lane_shift_32(n)) },
602 _ => unreachable!(),
603 }
604 }
605 pub fn right_lane_shift(self, n: u32) -> Self {
606 match T::BIT_SIZE {
607 BitSize::Size32 => unsafe { Self::new(self.data.right_lane_shift_32(n)) },
608 _ => unreachable!(),
609 }
610 }
611}