1use std::mem::transmute_copy;
2
3use crate::architectures::interface::*;
4
5#[derive(Copy, Clone)]
6#[repr(align(8))]
7pub struct ScalarReg<const N: usize>(pub [u8; N]);
8#[derive(Copy, Clone)]
9pub struct ScalarMask<const N: usize>(pub [bool; N]);
10
11impl<const N: usize> SimdArch for ScalarReg<N> {}
12impl<const N: usize> MaskArch for ScalarMask<N> {}
13
14macro_rules! scalar_token_op {
15 ($type:ty, $op:tt, $self:ident, $rhs:ident, $size:expr) => {
16 unsafe {
17 let mut new = ScalarReg::<$size>([0; $size]);
18
19 let self_ptr: *const $type = $self.0.as_ptr() as *const $type;
20 let rhs_ptr: *const $type = $rhs.0.as_ptr() as *const $type;
21 let new_ptr: *mut $type = new.0.as_mut_ptr() as *mut $type;
22
23 for i in 0..($size / size_of::<$type>()) {
24 *new_ptr.add(i) = *self_ptr.add(i) $op *rhs_ptr.add(i);
25 }
26
27 new
28 }
29 }
30}
31
32macro_rules! scalar_token_op_usize_rhs {
33 ($type:ty, $op:tt, $self:ident, $rhs:ident, $size:expr) => {
34 unsafe {
35 let mut new = ScalarReg::<$size>([0; $size]);
36
37 let self_ptr: *const $type = $self.0.as_ptr() as *const $type;
38 let rhs_ptr: *const $type = $rhs.0.as_ptr() as *const $type;
39 let new_ptr: *mut $type = new.0.as_mut_ptr() as *mut $type;
40
41 for i in 0..($size / size_of::<$type>()) {
42 *new_ptr.add(i) = *self_ptr.add(i) $op (*rhs_ptr.add(i) as usize);
43 }
44
45 new
46 }
47 }
48}
49
50macro_rules! scalar_func_op {
51 ($type:ty, $op:ident, $self:ident, $rhs:ident, $size:expr) => {
52 unsafe {
53 let mut new = ScalarReg::<$size>([0; $size]);
54
55 let self_ptr: *const $type = $self.0.as_ptr() as *const $type;
56 let rhs_ptr: *const $type = $rhs.0.as_ptr() as *const $type;
57 let new_ptr: *mut $type = new.0.as_mut_ptr() as *mut $type;
58
59 for i in 0..($size / size_of::<$type>()) {
60 *new_ptr.add(i) = (*self_ptr.add(i)).$op(*rhs_ptr.add(i));
61 }
62
63 new
64 }
65 };
66}
67
68macro_rules! scalar_self_op {
69 ($type:ty, $op:ident, $self:ident, $size:expr) => {
70 unsafe {
71 let mut new = ScalarReg::<$size>([0; $size]);
72
73 let self_ptr: *const $type = $self.0.as_ptr() as *const $type;
74 let new_ptr: *mut $type = new.0.as_mut_ptr() as *mut $type;
75
76 for i in 0..($size / size_of::<$type>()) {
77 *new_ptr.add(i) = (*self_ptr.add(i)).$op();
78 }
79 new
80 }
81 };
82}
83
84macro_rules! scalar_fma_expr_op {
85 ($type:ty, $self:ident, $mult:ident, $add:ident, $size:expr, |$a:ident, $b:ident, $c:ident| $op:expr) => {
86 unsafe {
87 let mut new = ScalarReg::<$size>([0; $size]);
88
89 let a_ptr: *const $type = $self.0.as_ptr() as *const $type;
90 let b_ptr: *const $type = $mult.0.as_ptr() as *const $type;
91 let c_ptr: *const $type = $add.0.as_ptr() as *const $type;
92 let new_ptr: *mut $type = new.0.as_mut_ptr() as *mut $type;
93
94 for i in 0..($size / size_of::<$type>()) {
95 let $a = *a_ptr.add(i);
96 let $b = *b_ptr.add(i);
97 let $c = *c_ptr.add(i);
98 *new_ptr.add(i) = $op;
99 }
100 new
101 }
102 };
103}
104
105macro_rules! scalar_cmp {
106 {$type:ty, $op:tt, $self:ident, $rhs:ident, $size:expr} => {
107 unsafe {
108 let mut result = ScalarMask::<$size>([false; $size]);
109
110 let self_ptr = $self.0.as_ptr() as *const $type;
111 let rhs_ptr = $rhs.0.as_ptr() as *const $type;
112
113 for i in 0..($size / size_of::<$type>()) {
114 result.0[i] = *self_ptr.add(i) $op *rhs_ptr.add(i);
115 }
116
117 result
118 }
119 }
120}
121
122macro_rules! scalar_splat {
123 {$type:ty, $self:ident, $val:expr, $size:expr} => {
124 unsafe {
125 let mut new = ScalarReg::<$size>([0; $size]);
126 let new_ptr = new.0.as_mut_ptr() as *mut $type;
127
128 for i in 0..(N / size_of::<$type>()) {
129 *new_ptr.add(i) = $val;
130 }
131
132 new
133 }
134 }
135}
136
137impl<const N: usize> SimdAddImpl for ScalarReg<N> {
138 #[inline(always)]
139 unsafe fn f64_add(self, rhs: Self) -> Self {
140 scalar_token_op!(f64, +, self, rhs, N)
141 }
142 #[inline(always)]
143 unsafe fn f32_add(self, rhs: Self) -> Self {
144 scalar_token_op!(f32, +, self, rhs, N)
145 }
146 #[inline(always)]
147 unsafe fn i64_add(self, rhs: Self) -> Self {
148 scalar_func_op!(i64, wrapping_add, self, rhs, N)
149 }
150 #[inline(always)]
151 unsafe fn i32_add(self, rhs: Self) -> Self {
152 scalar_func_op!(i32, wrapping_add, self, rhs, N)
153 }
154 #[inline(always)]
155 unsafe fn i16_add(self, rhs: Self) -> Self {
156 scalar_func_op!(i16, wrapping_add, self, rhs, N)
157 }
158 #[inline(always)]
159 unsafe fn i8_add(self, rhs: Self) -> Self {
160 scalar_func_op!(i8, wrapping_add, self, rhs, N)
161 }
162}
163
164impl<const N: usize> SimdSubImpl for ScalarReg<N> {
165 #[inline(always)]
166 unsafe fn f64_sub(self, rhs: Self) -> Self {
167 scalar_token_op!(f64, -, self, rhs, N)
168 }
169 #[inline(always)]
170 unsafe fn f32_sub(self, rhs: Self) -> Self {
171 scalar_token_op!(f32, -, self, rhs, N)
172 }
173 #[inline(always)]
174 unsafe fn i64_sub(self, rhs: Self) -> Self {
175 scalar_func_op!(i64, wrapping_sub, self, rhs, N)
176 }
177 #[inline(always)]
178 unsafe fn i32_sub(self, rhs: Self) -> Self {
179 scalar_func_op!(i32, wrapping_sub, self, rhs, N)
180 }
181 #[inline(always)]
182 unsafe fn i16_sub(self, rhs: Self) -> Self {
183 scalar_func_op!(i16, wrapping_sub, self, rhs, N)
184 }
185 #[inline(always)]
186 unsafe fn i8_sub(self, rhs: Self) -> Self {
187 scalar_func_op!(i8, wrapping_sub, self, rhs, N)
188 }
189}
190
191impl<const N: usize> SimdMulImpl for ScalarReg<N> {
192 #[inline(always)]
193 unsafe fn f64_mul(self, rhs: Self) -> Self {
194 scalar_token_op!(f64, *, self, rhs, N)
195 }
196 #[inline(always)]
197 unsafe fn f32_mul(self, rhs: Self) -> Self {
198 scalar_token_op!(f32, *, self, rhs, N)
199 }
200 #[inline(always)]
201 unsafe fn i32_mul(self, rhs: Self) -> Self {
202 scalar_func_op!(i32, wrapping_mul, self, rhs, N)
203 }
204 #[inline(always)]
205 unsafe fn i16_mul(self, rhs: Self) -> Self {
206 scalar_func_op!(i16, wrapping_mul, self, rhs, N)
207 }
208}
209
210impl<const N: usize> SimdDivImpl for ScalarReg<N> {
211 #[inline(always)]
212 unsafe fn f64_div(self, rhs: Self) -> Self {
213 scalar_token_op!(f64, /, self, rhs, N)
214 }
215 #[inline(always)]
216 unsafe fn f32_div(self, rhs: Self) -> Self {
217 scalar_token_op!(f32, /, self, rhs, N)
218 }
219}
220
221impl<const N: usize> SimdBitwiseImpl for ScalarReg<N> {
222 #[inline(always)]
223 unsafe fn and(self, rhs: Self) -> Self {
224 scalar_token_op!(u64, &, self, rhs, N)
225 }
226 #[inline(always)]
227 unsafe fn or(self, rhs: Self) -> Self {
228 scalar_token_op!(u64, |, self, rhs, N)
229 }
230 #[inline(always)]
231 unsafe fn xor(self, rhs: Self) -> Self {
232 scalar_token_op!(u64, ^, self, rhs, N)
233 }
234 #[inline(always)]
235 unsafe fn not(self) -> Self {
236 unsafe { self.xor(Self([255; N])) }
237 }
238 #[inline(always)]
239 unsafe fn and_not(self, rhs: Self) -> Self {
240 unsafe { self.and(rhs.not()) }
241 }
242}
243
244impl<const N: usize> SimdShiftImpl for ScalarReg<N> {
245 #[inline(always)]
246 unsafe fn sllv_64(self, rhs: Self) -> Self {
247 scalar_token_op_usize_rhs!(u64, <<, self, rhs, N)
248 }
249 #[inline(always)]
250 unsafe fn srlv_64(self, rhs: Self) -> Self {
251 scalar_token_op_usize_rhs!(u64, >>, self, rhs, N)
252 }
253 #[inline(always)]
254 unsafe fn srav_64(self, rhs: Self) -> Self {
255 scalar_token_op_usize_rhs!(i64, >>, self, rhs, N)
256 }
257 #[inline(always)]
258 unsafe fn sllv_32(self, rhs: Self) -> Self {
259 scalar_token_op_usize_rhs!(u32, <<, self, rhs, N)
260 }
261 #[inline(always)]
262 unsafe fn srlv_32(self, rhs: Self) -> Self {
263 scalar_token_op_usize_rhs!(u32, >>, self, rhs, N)
264 }
265 #[inline(always)]
266 unsafe fn srav_32(self, rhs: Self) -> Self {
267 scalar_token_op_usize_rhs!(i32, >>, self, rhs, N)
268 }
269 #[inline(always)]
270 unsafe fn sllv_16(self, rhs: Self) -> Self {
271 scalar_token_op_usize_rhs!(u16, <<, self, rhs, N)
272 }
273 #[inline(always)]
274 unsafe fn srlv_16(self, rhs: Self) -> Self {
275 scalar_token_op_usize_rhs!(u16, >>, self, rhs, N)
276 }
277 #[inline(always)]
278 unsafe fn srav_16(self, rhs: Self) -> Self {
279 scalar_token_op_usize_rhs!(i16, >>, self, rhs, N)
280 }
281}
282
283impl<const N: usize> SimdLoadImpl for ScalarReg<N> {
284 type MaskType = ScalarMask<N>;
285 #[inline(always)]
286 unsafe fn load_aligned<T>(ptr: *const T) -> Self {
287 unsafe {
288 let mut new = Self([0; N]);
289 std::ptr::copy_nonoverlapping(ptr as *const u8, new.0.as_mut_ptr(), N);
290 new
291 }
292 }
293 #[inline(always)]
294 unsafe fn load_unaligned<T>(ptr: *const T) -> Self {
295 unsafe { Self::load_aligned::<T>(ptr) }
296 }
297 #[inline(always)]
298 unsafe fn masked_load_64<T>(ptr: *const T, mask: Self::MaskType) -> Self {
299 unsafe {
300 let mut new = Self([0; N]);
301 let new_ptr = new.0.as_mut_ptr() as *mut u64;
302 let data_ptr = ptr as *const u64;
303 for i in 0..(N / 8) {
304 *new_ptr.add(i) = if mask.0[i] { *data_ptr.add(i) } else { 0 };
305 }
306 new
307 }
308 }
309 #[inline(always)]
310 unsafe fn masked_load_32<T>(ptr: *const T, mask: Self::MaskType) -> Self {
311 unsafe {
312 let mut new = Self([0; N]);
313 let new_ptr = new.0.as_mut_ptr() as *mut u32;
314 let data_ptr = ptr as *const u32;
315 for i in 0..(N / 4) {
316 *new_ptr.add(i) = if mask.0[i] { *data_ptr.add(i) } else { 0 };
317 }
318 new
319 }
320 }
321}
322
323impl<const N: usize> SimdStoreImpl for ScalarReg<N> {
324 type MaskType = ScalarMask<N>;
325 #[inline(always)]
326 unsafe fn store_aligned<T>(self, ptr: *mut T) {
327 unsafe {
328 std::ptr::copy_nonoverlapping(self.0.as_ptr(), ptr as *mut u8, N);
329 }
330 }
331 #[inline(always)]
332 unsafe fn store_unaligned<T>(self, ptr: *mut T) {
333 unsafe { self.store_aligned(ptr) };
334 }
335 #[inline(always)]
336 unsafe fn masked_store_64<T>(self, ptr: *mut T, mask: Self::MaskType) {
337 unsafe {
338 let self_ptr = self.0.as_ptr() as *const u64;
339 let store_ptr = ptr as *mut u64;
340 for i in 0..(N / 8) {
341 if mask.0[i] {
342 *store_ptr.add(i) = *self_ptr.add(i);
343 }
344 }
345 }
346 }
347 #[inline(always)]
348 unsafe fn masked_store_32<T>(self, ptr: *mut T, mask: Self::MaskType) {
349 unsafe {
350 let self_ptr = self.0.as_ptr() as *const u32;
351 let store_ptr = ptr as *mut u32;
352 for i in 0..(N / 4) {
353 if mask.0[i] {
354 *store_ptr.add(i) = *self_ptr.add(i);
355 }
356 }
357 }
358 }
359}
360
361impl<const N: usize> SimdZeroImpl for ScalarReg<N> {
362 #[inline(always)]
363 unsafe fn zero() -> Self {
364 Self([0; N])
365 }
366}
367
368impl<const N: usize> SimdFloatCastsImpl for ScalarReg<N> {
369 #[inline(always)]
370 unsafe fn float_to_int_trunc(self) -> Self {
371 unsafe {
372 let mut new = Self([0; N]);
373 let float_ptr = self.0.as_ptr() as *const f32;
374 let int_ptr = new.0.as_mut_ptr() as *mut i32;
375 for i in 0..(N / 4) {
376 *int_ptr.add(i) = (*float_ptr.add(i)) as i32;
377 }
378 new
379 }
380 }
381 #[inline(always)]
382 unsafe fn float_to_int_round(self) -> Self {
383 unsafe {
384 let mut new = Self([0; N]);
385 let float_ptr = self.0.as_ptr() as *const f32;
386 let int_ptr = new.0.as_mut_ptr() as *mut i32;
387 for i in 0..(N / 4) {
388 *int_ptr.add(i) = (*float_ptr.add(i)).round_ties_even() as i32;
389 }
390 new
391 }
392 }
393}
394
395impl<const N: usize> SimdIntCastsImpl for ScalarReg<N> {
396 #[inline(always)]
397 unsafe fn int_to_float(self) -> Self {
398 unsafe {
399 let mut new = Self([0; N]);
400 let int_ptr = self.0.as_ptr() as *const i32;
401 let float_ptr = new.0.as_mut_ptr() as *mut f32;
402 for i in 0..(N / 4) {
403 *float_ptr.add(i) = (*int_ptr.add(i)) as f32;
404 }
405 new
406 }
407 }
408}
409
410impl<const N: usize> SimdPermuteImpl for ScalarReg<N> {
411 #[inline(always)]
412 unsafe fn permute_32(self, rhs: Self) -> Self {
413 unsafe {
414 let mut new = Self([0; N]);
415 let new_ptr = new.0.as_mut_ptr() as *mut u32;
416 let self_ptr = self.0.as_ptr() as *const u32;
417 let indices_ptr = rhs.0.as_ptr() as *const u32;
418 for i in 0..(N / 4) {
419 let index = *indices_ptr.add(i) as usize;
420 *new_ptr.add(i) = if index < (N / 4) {
421 *self_ptr.add(index)
422 } else {
423 0
424 };
425 }
426 new
427 }
428 }
429 #[inline(always)]
430 unsafe fn permute_8(self, rhs: Self) -> Self {
431 unsafe {
432 let mut new = Self([0; N]);
433 let new_ptr = new.0.as_mut_ptr();
434 let self_ptr = self.0.as_ptr();
435 let indices_ptr = rhs.0.as_ptr();
436 for i in 0..N {
437 let lane_base = i & !15; let index = (*indices_ptr.add(i) as usize & 15) + lane_base;
439 *new_ptr.add(i) = *self_ptr.add(index);
440 }
441 new
442 }
443 }
444}
445
446impl<const N: usize> SimdVariableBlendImpl for ScalarMask<N> {
447 type VecType = ScalarReg<N>;
448 #[inline(always)]
449 unsafe fn vblend_64(
450 self,
451 true_values: Self::VecType,
452 false_values: Self::VecType,
453 ) -> ScalarReg<N> {
454 unsafe {
455 let mut new = ScalarReg::<N>([0; N]);
456 let new_ptr = new.0.as_mut_ptr() as *mut u64;
457 let false_ptr = false_values.0.as_ptr() as *const u64;
458 let true_ptr = true_values.0.as_ptr() as *const u64;
459 for i in 0..(N / 8) {
460 *new_ptr.add(i) = if self.0[i] {
461 *true_ptr.add(i)
462 } else {
463 *false_ptr.add(i)
464 };
465 }
466 new
467 }
468 }
469 #[inline(always)]
470 unsafe fn vblend_32(
471 self,
472 true_values: Self::VecType,
473 false_values: Self::VecType,
474 ) -> ScalarReg<N> {
475 unsafe {
476 let mut new = ScalarReg::<N>([0; N]);
477 let new_ptr = new.0.as_mut_ptr() as *mut u32;
478 let false_ptr = false_values.0.as_ptr() as *const u32;
479 let true_ptr = true_values.0.as_ptr() as *const u32;
480 for i in 0..(N / 4) {
481 *new_ptr.add(i) = if self.0[i] {
482 *true_ptr.add(i)
483 } else {
484 *false_ptr.add(i)
485 };
486 }
487 new
488 }
489 }
490 #[inline(always)]
491 unsafe fn vblend_8(
492 self,
493 true_values: Self::VecType,
494 false_values: Self::VecType,
495 ) -> ScalarReg<N> {
496 unsafe {
497 let mut new = ScalarReg::<N>([0; N]);
498 let new_ptr = new.0.as_mut_ptr();
499 let false_ptr = false_values.0.as_ptr();
500 let true_ptr = true_values.0.as_ptr();
501 for i in 0..N {
502 *new_ptr.add(i) = if self.0[i] {
503 *true_ptr.add(i)
504 } else {
505 *false_ptr.add(i)
506 };
507 }
508 new
509 }
510 }
511}
512
513impl<const M: usize> SimdImmediateBlendImpl for ScalarReg<M> {
514 #[inline(always)]
515 unsafe fn blend_64<const N: i32>(self, false_values: Self) -> Self {
516 unsafe {
517 let mut new = ScalarReg::<M>([0; M]);
518 let new_ptr = new.0.as_mut_ptr() as *mut u64;
519 let false_ptr = false_values.0.as_ptr() as *const u64;
520 let true_ptr = self.0.as_ptr() as *const u64;
521 for i in 0..(M >> 3) {
522 let cond = ((N >> i) & 1) == 1;
523 *new_ptr.add(i) = if cond {
524 *true_ptr.add(i)
525 } else {
526 *false_ptr.add(i)
527 };
528 }
529 new
530 }
531 }
532 #[inline(always)]
533 unsafe fn blend_32<const N: i32>(self, false_values: Self) -> Self {
534 unsafe {
535 let mut new = ScalarReg::<M>([0; M]);
536 let new_ptr = new.0.as_mut_ptr() as *mut u32;
537 let false_ptr = false_values.0.as_ptr() as *const u32;
538 let true_ptr = self.0.as_ptr() as *const u32;
539 for i in 0..(M >> 2) {
540 let cond = ((N >> i) & 1) == 1;
541 *new_ptr.add(i) = if cond {
542 *true_ptr.add(i)
543 } else {
544 *false_ptr.add(i)
545 };
546 }
547 new
548 }
549 }
550}
551
552impl<const N: usize> SimdMulAddImpl for ScalarReg<N> {
553 #[inline(always)]
554 unsafe fn mul_add_f64(self, mult: Self, add: Self) -> Self {
555 scalar_fma_expr_op!(f64, self, mult, add, N, |a, b, c| f64::mul_add(a, b, c))
556 }
557 #[inline(always)]
558 unsafe fn mul_sub_f64(self, mult: Self, sub: Self) -> Self {
559 scalar_fma_expr_op!(f64, self, mult, sub, N, |a, b, c| f64::mul_add(a, b, -c))
560 }
561 #[inline(always)]
562 unsafe fn negated_mul_add_f64(self, mult: Self, add: Self) -> Self {
563 scalar_fma_expr_op!(f64, self, mult, add, N, |a, b, c| f64::mul_add(-a, b, c))
564 }
565 #[inline(always)]
566 unsafe fn negated_mul_sub_f64(self, mult: Self, sub: Self) -> Self {
567 scalar_fma_expr_op!(f64, self, mult, sub, N, |a, b, c| f64::mul_add(-a, b, -c))
568 }
569 #[inline(always)]
570 unsafe fn mul_add_f32(self, mult: Self, add: Self) -> Self {
571 scalar_fma_expr_op!(f32, self, mult, add, N, |a, b, c| f32::mul_add(a, b, c))
572 }
573 #[inline(always)]
574 unsafe fn mul_sub_f32(self, mult: Self, sub: Self) -> Self {
575 scalar_fma_expr_op!(f32, self, mult, sub, N, |a, b, c| f32::mul_add(a, b, -c))
576 }
577 #[inline(always)]
578 unsafe fn negated_mul_add_f32(self, mult: Self, add: Self) -> Self {
579 scalar_fma_expr_op!(f32, self, mult, add, N, |a, b, c| f32::mul_add(-a, b, c))
580 }
581 #[inline(always)]
582 unsafe fn negated_mul_sub_f32(self, mult: Self, sub: Self) -> Self {
583 scalar_fma_expr_op!(f32, self, mult, sub, N, |a, b, c| f32::mul_add(-a, b, -c))
584 }
585}
586
587impl<const N: usize> SimdRoundImpl for ScalarReg<N> {
588 #[inline(always)]
589 unsafe fn round_f64(self) -> Self {
590 scalar_self_op!(f64, round_ties_even, self, N)
591 }
592 #[inline(always)]
593 unsafe fn round_f32(self) -> Self {
594 scalar_self_op!(f32, round_ties_even, self, N)
595 }
596 #[inline(always)]
597 unsafe fn floor_f64(self) -> Self {
598 scalar_self_op!(f64, floor, self, N)
599 }
600 #[inline(always)]
601 unsafe fn floor_f32(self) -> Self {
602 scalar_self_op!(f32, floor, self, N)
603 }
604 #[inline(always)]
605 unsafe fn ceil_f64(self) -> Self {
606 scalar_self_op!(f64, ceil, self, N)
607 }
608 #[inline(always)]
609 unsafe fn ceil_f32(self) -> Self {
610 scalar_self_op!(f32, ceil, self, N)
611 }
612}
613
614impl<const N: usize> SimdPartialOrdImpl for ScalarReg<N> {
615 type MaskType = ScalarMask<N>;
616 #[inline(always)]
617 unsafe fn cmp_f64_eq(self, rhs: Self) -> Self::MaskType {
618 scalar_cmp!(f64, ==, self, rhs, N)
619 }
620 #[inline(always)]
621 unsafe fn cmp_f64_lt(self, rhs: Self) -> Self::MaskType {
622 scalar_cmp!(f64, <, self, rhs, N)
623 }
624 #[inline(always)]
625 unsafe fn cmp_f64_le(self, rhs: Self) -> Self::MaskType {
626 scalar_cmp!(f64, <=, self, rhs, N)
627 }
628 #[inline(always)]
629 unsafe fn cmp_f64_gt(self, rhs: Self) -> Self::MaskType {
630 scalar_cmp!(f64, >, self, rhs, N)
631 }
632 #[inline(always)]
633 unsafe fn cmp_f64_ge(self, rhs: Self) -> Self::MaskType {
634 scalar_cmp!(f64, >=, self, rhs, N)
635 }
636 #[inline(always)]
637 unsafe fn cmp_f64_neq(self, rhs: Self) -> Self::MaskType {
638 scalar_cmp!(f64, !=, self, rhs, N)
639 }
640 #[inline(always)]
641 unsafe fn cmp_f32_eq(self, rhs: Self) -> Self::MaskType {
642 scalar_cmp!(f32, ==, self, rhs, N)
643 }
644 #[inline(always)]
645 unsafe fn cmp_f32_lt(self, rhs: Self) -> Self::MaskType {
646 scalar_cmp!(f32, <, self, rhs, N)
647 }
648 #[inline(always)]
649 unsafe fn cmp_f32_le(self, rhs: Self) -> Self::MaskType {
650 scalar_cmp!(f32, <=, self, rhs, N)
651 }
652 #[inline(always)]
653 unsafe fn cmp_f32_gt(self, rhs: Self) -> Self::MaskType {
654 scalar_cmp!(f32, >, self, rhs, N)
655 }
656 #[inline(always)]
657 unsafe fn cmp_f32_ge(self, rhs: Self) -> Self::MaskType {
658 scalar_cmp!(f32, >=, self, rhs, N)
659 }
660 #[inline(always)]
661 unsafe fn cmp_f32_neq(self, rhs: Self) -> Self::MaskType {
662 scalar_cmp!(f32, !=, self, rhs, N)
663 }
664 #[inline(always)]
665 unsafe fn cmp_i64_eq(self, rhs: Self) -> Self::MaskType {
666 scalar_cmp!(i64, ==, self, rhs, N)
667 }
668 #[inline(always)]
669 unsafe fn cmp_i64_gt(self, rhs: Self) -> Self::MaskType {
670 scalar_cmp!(i64, >, self, rhs, N)
671 }
672 #[inline(always)]
673 unsafe fn cmp_i32_eq(self, rhs: Self) -> Self::MaskType {
674 scalar_cmp!(i32, ==, self, rhs, N)
675 }
676 #[inline(always)]
677 unsafe fn cmp_i32_gt(self, rhs: Self) -> Self::MaskType {
678 scalar_cmp!(i32, >, self, rhs, N)
679 }
680 #[inline(always)]
681 unsafe fn cmp_i16_eq(self, rhs: Self) -> Self::MaskType {
682 scalar_cmp!(i16, ==, self, rhs, N)
683 }
684 #[inline(always)]
685 unsafe fn cmp_i16_gt(self, rhs: Self) -> Self::MaskType {
686 scalar_cmp!(i16, >, self, rhs, N)
687 }
688 #[inline(always)]
689 unsafe fn cmp_i8_eq(self, rhs: Self) -> Self::MaskType {
690 scalar_cmp!(i8, ==, self, rhs, N)
691 }
692 #[inline(always)]
693 unsafe fn cmp_i8_gt(self, rhs: Self) -> Self::MaskType {
694 scalar_cmp!(i8, >, self, rhs, N)
695 }
696
697 #[inline(always)]
698 unsafe fn max_f64(self, rhs: Self) -> Self {
699 scalar_func_op!(f64, max, self, rhs, N)
700 }
701 #[inline(always)]
702 unsafe fn min_f64(self, rhs: Self) -> Self {
703 scalar_func_op!(f64, min, self, rhs, N)
704 }
705 #[inline(always)]
706 unsafe fn max_f32(self, rhs: Self) -> Self {
707 scalar_func_op!(f32, max, self, rhs, N)
708 }
709 #[inline(always)]
710 unsafe fn min_f32(self, rhs: Self) -> Self {
711 scalar_func_op!(f32, min, self, rhs, N)
712 }
713 #[inline(always)]
714 unsafe fn max_i32(self, rhs: Self) -> Self {
715 scalar_func_op!(i32, max, self, rhs, N)
716 }
717 #[inline(always)]
718 unsafe fn min_i32(self, rhs: Self) -> Self {
719 scalar_func_op!(i32, min, self, rhs, N)
720 }
721 #[inline(always)]
722 unsafe fn max_i16(self, rhs: Self) -> Self {
723 scalar_func_op!(i16, max, self, rhs, N)
724 }
725 #[inline(always)]
726 unsafe fn min_i16(self, rhs: Self) -> Self {
727 scalar_func_op!(i16, min, self, rhs, N)
728 }
729 #[inline(always)]
730 unsafe fn max_i8(self, rhs: Self) -> Self {
731 scalar_func_op!(i8, max, self, rhs, N)
732 }
733 #[inline(always)]
734 unsafe fn min_i8(self, rhs: Self) -> Self {
735 scalar_func_op!(i8, min, self, rhs, N)
736 }
737 #[inline(always)]
738 unsafe fn max_u32(self, rhs: Self) -> Self {
739 scalar_func_op!(u32, max, self, rhs, N)
740 }
741 #[inline(always)]
742 unsafe fn min_u32(self, rhs: Self) -> Self {
743 scalar_func_op!(u32, min, self, rhs, N)
744 }
745 #[inline(always)]
746 unsafe fn max_u16(self, rhs: Self) -> Self {
747 scalar_func_op!(u16, max, self, rhs, N)
748 }
749 #[inline(always)]
750 unsafe fn min_u16(self, rhs: Self) -> Self {
751 scalar_func_op!(u16, min, self, rhs, N)
752 }
753 #[inline(always)]
754 unsafe fn max_u8(self, rhs: Self) -> Self {
755 scalar_func_op!(u8, max, self, rhs, N)
756 }
757 #[inline(always)]
758 unsafe fn min_u8(self, rhs: Self) -> Self {
759 scalar_func_op!(u8, min, self, rhs, N)
760 }
761}
762
763impl<const N: usize> SimdSplatImpl for ScalarReg<N> {
765 #[inline(always)]
766 unsafe fn splat_64<T>(val: T) -> Self {
767 scalar_splat!(u64, self, transmute_copy(&val), N)
768 }
769 #[inline(always)]
770 unsafe fn splat_32<T>(val: T) -> Self {
771 scalar_splat!(u32, self, transmute_copy(&val), N)
772 }
773 #[inline(always)]
774 unsafe fn splat_16<T>(val: T) -> Self {
775 scalar_splat!(u16, self, transmute_copy(&val), N)
776 }
777 #[inline(always)]
778 unsafe fn splat_8<T>(val: T) -> Self {
779 scalar_splat!(u8, self, transmute_copy(&val), N)
780 }
781}
782
783impl<const N: usize> SimdGatherImpl for ScalarReg<N> {
784 #[inline(always)]
785 unsafe fn gather_32_from_32<T, const B: i32>(self, ptr: *const T) -> Self {
786 unsafe {
787 let mut new = Self([0; N]);
788 let new_ptr = new.0.as_mut_ptr() as *mut u32;
789 let indices_ptr = self.0.as_ptr() as *const u32;
790 let data_ptr = ptr as *const u32;
791 for i in 0..(N / 4) {
792 let index = *indices_ptr.add(i) as usize;
793 *new_ptr.add(i) = *data_ptr.add(index);
794 }
795 new
796 }
797 }
798 #[inline(always)]
799 unsafe fn gather_64_from_64<T, const B: i32>(self, ptr: *const T) -> Self {
800 unsafe {
801 let mut new = Self([0; N]);
802 let new_ptr = new.0.as_mut_ptr() as *mut u64;
803 let indices_ptr = self.0.as_ptr() as *const u64;
804 let data_ptr = ptr as *const u64;
805 for i in 0..(N / 8) {
806 let index = *indices_ptr.add(i) as usize;
807 *new_ptr.add(i) = *data_ptr.add(index);
808 }
809 new
810 }
811 }
812}
813
814impl<const N: usize> SimdSqrtImpl for ScalarReg<N> {
815 #[inline(always)]
816 unsafe fn sqrt_f64(self) -> Self {
817 scalar_self_op!(f64, sqrt, self, N)
818 }
819 #[inline(always)]
820 unsafe fn sqrt_f32(self) -> Self {
821 scalar_self_op!(f32, sqrt, self, N)
822 }
823 #[inline(always)]
824 unsafe fn rsqrt_f32(self) -> Self {
825 unsafe { Self::splat_32(1.0f32).f32_div(self.sqrt_f32()) }
826 }
827}
828
829impl<const N: usize> SimdAllBitsImpl for ScalarMask<N> {
830 #[inline(always)]
831 unsafe fn all_zero(self) -> bool {
832 !self.0.iter().any(|&x| x)
833 }
834}
835
836impl<const N: usize> SimdBitwiseImpl for ScalarMask<N> {
837 #[inline(always)]
838 unsafe fn and(self, rhs: Self) -> Self {
839 Self(std::array::from_fn(|i| self.0[i] & rhs.0[i]))
840 }
841 #[inline(always)]
842 unsafe fn or(self, rhs: Self) -> Self {
843 Self(std::array::from_fn(|i| self.0[i] | rhs.0[i]))
844 }
845 #[inline(always)]
846 unsafe fn xor(self, rhs: Self) -> Self {
847 Self(std::array::from_fn(|i| self.0[i] ^ rhs.0[i]))
848 }
849 #[inline(always)]
850 unsafe fn not(self) -> Self {
851 Self(std::array::from_fn(|i| !self.0[i]))
852 }
853 #[inline(always)]
854 unsafe fn and_not(self, rhs: Self) -> Self {
855 unsafe { self.and(rhs.not()) }
856 }
857}
858
859impl<const N: usize> SimdNegateImpl for ScalarReg<N> {
860 #[inline(always)]
861 unsafe fn negate_f64(self) -> Self {
862 unsafe { Self::splat_64(-0.0f64).xor(self) }
863 }
864 #[inline(always)]
865 unsafe fn negate_f32(self) -> Self {
866 unsafe { Self::splat_32(-0.0f64).xor(self) }
867 }
868}
869
870impl<const N: usize> SimdBlockShiftImpl for ScalarReg<N> {
871 #[inline(always)]
872 unsafe fn block_left_byte_shift<const M: i32>(self) -> Self {
873 let mut new = unsafe { Self::splat_8(0) };
874 for block_start in (0..N).step_by(16) {
875 let block_end = block_start + 16;
876 for i in (block_start + M as usize)..block_end {
877 new.0[i] = self.0[i - M as usize];
878 }
879 }
880 new
881 }
882 #[inline(always)]
883 unsafe fn block_right_byte_shift<const M: i32>(self) -> Self {
884 let mut new = unsafe { Self::splat_8(0) };
885 for block_start in (0..N).step_by(16) {
886 let block_end = block_start + 16;
887 for i in block_start..(block_end - M as usize) {
888 new.0[i] = self.0[i + M as usize];
889 }
890 }
891 new
892 }
893}
894
895impl<const N: usize> SimdMaskBitConversion for ScalarMask<N> {
896 #[inline(always)]
897 unsafe fn to_bits_64(self) -> u64 {
898 let mut bits = 0u64;
899 for i in 0..(N >> 3) {
900 bits ^= (self.0[i] as u64) << i
901 }
902 bits
903 }
904 #[inline(always)]
905 unsafe fn to_bits_32(self) -> u64 {
906 let mut bits = 0u64;
907 for i in 0..(N >> 2) {
908 bits ^= (self.0[i] as u64) << i
909 }
910 bits
911 }
912 #[inline(always)]
913 unsafe fn to_bits_8(self) -> u64 {
914 let mut bits = 0u64;
915 for i in 0..N {
916 bits ^= (self.0[i] as u64) << i
917 }
918 bits
919 }
920 #[inline(always)]
921 unsafe fn from_bits_64(bitmask: u64) -> Self {
922 let mut new_mask = Self([false; N]);
923 for i in 0..N {
924 new_mask.0[i] = ((bitmask >> i) & 1) == 1;
925 }
926 new_mask
927 }
928 unsafe fn from_bits_32(bitmask: u64) -> Self {
929 let mut new_mask = Self([false; N]);
930 for i in 0..N {
931 new_mask.0[i] = ((bitmask >> i) & 1) == 1;
932 }
933 new_mask
934 }
935 unsafe fn from_bits_16(bitmask: u64) -> Self {
936 let mut new_mask = Self([false; N]);
937 for i in 0..N {
938 new_mask.0[i] = ((bitmask >> i) & 1) == 1;
939 }
940 new_mask
941 }
942 unsafe fn from_bits_8(bitmask: u64) -> Self {
943 let mut new_mask = Self([false; N]);
944 for i in 0..N {
945 new_mask.0[i] = ((bitmask >> i) & 1) == 1;
946 }
947 new_mask
948 }
949}
950
951impl<const M: usize> SimdLaneShiftImpl for ScalarReg<M> {
952 #[inline(always)]
953 unsafe fn left_lane_shift_32(self, n: u32) -> Self {
954 let mut new = unsafe { Self::zero() };
955 if n as usize * 4 >= M {
956 new
957 } else {
958 let bytes = (n * 4) as usize;
959 for i in 0..(M - bytes) {
960 new.0[i] = self.0[i + bytes];
961 }
962 new
963 }
964 }
965
966 #[inline(always)]
967 unsafe fn right_lane_shift_32(self, n: u32) -> Self {
968 let mut new = unsafe { Self::zero() };
969 if n as usize * 4 >= M {
970 new
971 } else {
972 let bytes = (n * 4) as usize;
973 for i in bytes..M {
974 new.0[i] = self.0[i - bytes];
975 }
976 new
977 }
978 }
979}