1#[inline(always)]
2pub(crate) fn dispatch<R>(f: impl FnOnce() -> R) -> R {
3 #[cfg(feature = "simd")]
4 {
5 pulp::Arch::new().dispatch(f)
6 }
7 #[cfg(not(feature = "simd"))]
8 {
9 f()
10 }
11}
12
13#[inline(always)]
14pub(crate) fn dispatch_if_large<R>(len: usize, f: impl FnOnce() -> R) -> R {
15 if len >= 64 {
18 dispatch(f)
19 } else {
20 f()
21 }
22}
23
24#[cfg(feature = "simd")]
25#[inline(always)]
26unsafe fn cast_slice<T, U>(src: &[T]) -> &[U] {
27 debug_assert_eq!(std::mem::size_of::<T>(), std::mem::size_of::<U>());
28 unsafe { std::slice::from_raw_parts(src.as_ptr().cast::<U>(), src.len()) }
29}
30
31#[cfg(feature = "simd")]
32macro_rules! impl_simd_mul_ptr {
33 (
34 $mul_into:ident,
35 $ty:ty,
36 $split:ident,
37 $split_uninit:ident,
38 $load:ident,
39 $mask:ident,
40 $store_ptr:ident,
41 $mul:ident,
42 $mask_scale:expr
43 ) => {
44 unsafe fn $mul_into(dst: *mut $ty, len: usize, a: &[$ty], b: &[$ty]) {
45 struct Mul<'a> {
46 dst: *mut $ty,
47 len: usize,
48 a: &'a [$ty],
49 b: &'a [$ty],
50 }
51
52 impl<'a> pulp::WithSimd for Mul<'a> {
53 type Output = ();
54
55 #[inline(always)]
56 fn with_simd<S: pulp::Simd>(self, simd: S) -> Self::Output {
57 debug_assert_eq!(self.len, self.a.len());
58 debug_assert_eq!(self.len, self.b.len());
59
60 let output = unsafe {
63 core::slice::from_raw_parts_mut(
64 self.dst.cast::<core::mem::MaybeUninit<$ty>>(),
65 self.len,
66 )
67 };
68 let (dest, tail) = S::$split_uninit(output);
69 let (a, a_tail) = S::$split(self.a);
70 let (b, b_tail) = S::$split(self.b);
71 for ((dst, &a), &b) in dest.iter_mut().zip(a).zip(b) {
74 dst.write(simd.$mul(a, b));
75 }
76 if !tail.is_empty() {
77 let va = simd.$load(a_tail);
78 let vb = simd.$load(b_tail);
79 unsafe {
81 simd.$store_ptr(
82 simd.$mask(0, (tail.len() * $mask_scale) as _),
83 tail.as_mut_ptr().cast::<$ty>(),
84 simd.$mul(va, vb),
85 );
86 }
87 }
88 }
89 }
90
91 pulp::Arch::new().dispatch(Mul { dst, len, a, b });
92 }
93 };
94}
95
96#[cfg(feature = "simd")]
97impl_simd_mul_ptr!(
98 simd_mul_f32_into,
99 f32,
100 as_simd_f32s,
101 as_uninit_mut_simd_f32s,
102 partial_load_f32s,
103 mask_between_m32s,
104 mask_store_ptr_f32s,
105 mul_f32s,
106 1
107);
108
109#[cfg(feature = "simd")]
110impl_simd_mul_ptr!(
111 simd_mul_f64_into,
112 f64,
113 as_simd_f64s,
114 as_uninit_mut_simd_f64s,
115 partial_load_f64s,
116 mask_between_m64s,
117 mask_store_ptr_f64s,
118 mul_f64s,
119 1
120);
121
122#[cfg(feature = "simd")]
123impl_simd_mul_ptr!(
124 simd_mul_c32_into,
125 num_complex::Complex32,
126 as_simd_c32s,
127 as_uninit_mut_simd_c32s,
128 partial_load_c32s,
129 mask_between_m32s,
130 mask_store_ptr_c32s,
131 mul_e_c32s,
132 2
133);
134
135#[cfg(feature = "simd")]
136impl_simd_mul_ptr!(
137 simd_mul_c64_into,
138 num_complex::Complex64,
139 as_simd_c64s,
140 as_uninit_mut_simd_c64s,
141 partial_load_c64s,
142 mask_between_m64s,
143 mask_store_ptr_c64s,
144 mul_e_c64s,
145 2
146);
147
148#[cfg(feature = "simd")]
149#[inline]
150#[cfg_attr(not(test), allow(dead_code))]
152pub(crate) fn try_mul_contiguous<D: 'static, A: 'static, B: 'static>(
153 dst: &mut [D],
154 a: &[A],
155 b: &[B],
156) -> bool {
157 unsafe { try_mul_contiguous_ptr(dst.as_mut_ptr(), dst.len(), a, b) }
158}
159
160#[cfg(feature = "simd")]
161#[inline]
162pub(crate) unsafe fn try_mul_contiguous_ptr<D: 'static, A: 'static, B: 'static>(
163 dst: *mut D,
164 len: usize,
165 a: &[A],
166 b: &[B],
167) -> bool {
168 use std::any::TypeId;
169
170 macro_rules! try_same_type {
171 ($ty:ty, $mul_into:ident) => {
172 if TypeId::of::<D>() == TypeId::of::<$ty>()
173 && TypeId::of::<A>() == TypeId::of::<$ty>()
174 && TypeId::of::<B>() == TypeId::of::<$ty>()
175 {
176 unsafe { $mul_into(dst.cast::<$ty>(), len, cast_slice(a), cast_slice(b)) };
177 return true;
178 }
179 };
180 }
181
182 try_same_type!(f64, simd_mul_f64_into);
183 try_same_type!(f32, simd_mul_f32_into);
184 try_same_type!(num_complex::Complex64, simd_mul_c64_into);
185 try_same_type!(num_complex::Complex32, simd_mul_c32_into);
186
187 false
188}
189
190#[cfg(not(feature = "simd"))]
191#[inline]
192#[cfg_attr(not(test), allow(dead_code))]
194pub(crate) fn try_mul_contiguous<D: 'static, A: 'static, B: 'static>(
195 _dst: &mut [D],
196 _a: &[A],
197 _b: &[B],
198) -> bool {
199 false
200}
201
202#[cfg(not(feature = "simd"))]
203#[inline]
204pub(crate) unsafe fn try_mul_contiguous_ptr<D: 'static, A: 'static, B: 'static>(
205 _dst: *mut D,
206 _len: usize,
207 _a: &[A],
208 _b: &[B],
209) -> bool {
210 false
211}
212
213const TRANSPOSE_TILE: usize = 8;
214
215#[inline(always)]
216unsafe fn mul_transposed_scalar_rhs_source_contiguous<T>(
217 dst: *mut T,
218 src: *const T,
219 scalar: T,
220 inner_len: usize,
221 row_len: usize,
222 src_fast_stride: isize,
223 src_row_stride: isize,
224) where
225 T: Copy + std::ops::Mul<Output = T>,
226{
227 let mut inner0 = 0usize;
228
229 while inner0 < inner_len {
230 let inner_count = TRANSPOSE_TILE.min(inner_len - inner0);
231 let mut row0 = 0usize;
232
233 while row0 < row_len {
234 let row_count = TRANSPOSE_TILE.min(row_len - row0);
235
236 for inner in 0..inner_count {
237 let src_base = src.offset((inner0 + inner) as isize * src_fast_stride);
238 for row in 0..row_count {
239 let row_index = row0 + row;
240 let src_offset = row_index as isize * src_row_stride;
241 *dst.add(row_index * inner_len + inner0 + inner) =
242 *src_base.offset(src_offset) * scalar;
243 }
244 }
245
246 row0 += TRANSPOSE_TILE;
247 }
248
249 inner0 += TRANSPOSE_TILE;
250 }
251}
252
253#[inline(always)]
254unsafe fn mul_transposed_scalar_lhs_source_contiguous<T>(
255 dst: *mut T,
256 scalar: T,
257 src: *const T,
258 inner_len: usize,
259 row_len: usize,
260 src_fast_stride: isize,
261 src_row_stride: isize,
262) where
263 T: Copy + std::ops::Mul<Output = T>,
264{
265 let mut inner0 = 0usize;
266
267 while inner0 < inner_len {
268 let inner_count = TRANSPOSE_TILE.min(inner_len - inner0);
269 let mut row0 = 0usize;
270
271 while row0 < row_len {
272 let row_count = TRANSPOSE_TILE.min(row_len - row0);
273
274 for inner in 0..inner_count {
275 let src_base = src.offset((inner0 + inner) as isize * src_fast_stride);
276 for row in 0..row_count {
277 let row_index = row0 + row;
278 let src_offset = row_index as isize * src_row_stride;
279 *dst.add(row_index * inner_len + inner0 + inner) =
280 scalar * *src_base.offset(src_offset);
281 }
282 }
283
284 row0 += TRANSPOSE_TILE;
285 }
286
287 inner0 += TRANSPOSE_TILE;
288 }
289}
290
291#[inline]
292pub(crate) unsafe fn try_mul_transposed_scalar_rhs_2d<D: 'static, A: 'static, B: 'static>(
293 dst: *mut D,
294 src: *const A,
295 scalar: *const B,
296 inner_len: usize,
297 row_len: usize,
298 src_fast_stride: isize,
299 src_row_stride: isize,
300) -> bool {
301 use std::any::TypeId;
302
303 macro_rules! try_same_type {
304 ($ty:ty) => {
305 if TypeId::of::<D>() == TypeId::of::<$ty>()
306 && TypeId::of::<A>() == TypeId::of::<$ty>()
307 && TypeId::of::<B>() == TypeId::of::<$ty>()
308 {
309 dispatch_if_large(inner_len * row_len, || unsafe {
311 mul_transposed_scalar_rhs_source_contiguous(
312 dst.cast::<$ty>(),
313 src.cast::<$ty>(),
314 *scalar.cast::<$ty>(),
315 inner_len,
316 row_len,
317 src_fast_stride,
318 src_row_stride,
319 );
320 });
321 return true;
322 }
323 };
324 }
325
326 try_same_type!(f64);
327 try_same_type!(f32);
328 try_same_type!(num_complex::Complex64);
329 try_same_type!(num_complex::Complex32);
330
331 false
332}
333
334#[inline]
335pub(crate) unsafe fn try_mul_transposed_scalar_lhs_2d<D: 'static, A: 'static, B: 'static>(
336 dst: *mut D,
337 scalar: *const A,
338 src: *const B,
339 inner_len: usize,
340 row_len: usize,
341 src_fast_stride: isize,
342 src_row_stride: isize,
343) -> bool {
344 use std::any::TypeId;
345
346 macro_rules! try_same_type {
347 ($ty:ty) => {
348 if TypeId::of::<D>() == TypeId::of::<$ty>()
349 && TypeId::of::<A>() == TypeId::of::<$ty>()
350 && TypeId::of::<B>() == TypeId::of::<$ty>()
351 {
352 dispatch_if_large(inner_len * row_len, || unsafe {
354 mul_transposed_scalar_lhs_source_contiguous(
355 dst.cast::<$ty>(),
356 *scalar.cast::<$ty>(),
357 src.cast::<$ty>(),
358 inner_len,
359 row_len,
360 src_fast_stride,
361 src_row_stride,
362 );
363 });
364 return true;
365 }
366 };
367 }
368
369 try_same_type!(f64);
370 try_same_type!(f32);
371 try_same_type!(num_complex::Complex64);
372 try_same_type!(num_complex::Complex32);
373
374 false
375}
376
377pub trait MaybeSimdOps: Copy + Sized {
382 fn try_simd_sum(_src: &[Self]) -> Option<Self> {
383 None
384 }
385 fn try_simd_dot(_a: &[Self], _b: &[Self]) -> Option<Self> {
386 None
387 }
388}
389
390pub(crate) trait MaybeSimdProduct: Copy + Sized {
391 fn try_simd_product(_src: &[Self]) -> Option<Self> {
392 None
393 }
394}
395
396pub(crate) trait MaybeSimdSumSquares: Copy + Sized {
397 fn try_simd_sum_squares(_src: &[Self]) -> Option<Self> {
398 None
399 }
400}
401
402#[cfg(feature = "simd")]
404const BLOCK_FOLD_MIN_LEN: usize = 16;
405
406pub(crate) trait MaybeSimdBlockFold: Copy + Sized {
414 fn try_simd_add_block(_acc: &mut [Self], _values: &[Self]) -> bool {
415 false
416 }
417 fn try_simd_mul_block(_acc: &mut [Self], _values: &[Self]) -> bool {
418 false
419 }
420 fn try_simd_sum_squares_block(_acc: &mut [Self], _values: &[Self]) -> bool {
421 false
422 }
423}
424
425macro_rules! impl_no_simd {
427 ($($t:ty),*) => {
428 $(
429 impl MaybeSimdOps for $t {}
430 impl MaybeSimdProduct for $t {}
431 impl MaybeSimdSumSquares for $t {}
432 impl MaybeSimdBlockFold for $t {}
433 )*
434 };
435}
436
437impl_no_simd!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize);
438
439impl<T: num_traits::Num + Copy + Clone + std::ops::Neg<Output = T>> MaybeSimdOps
440 for num_complex::Complex<T>
441{
442}
443impl<T: num_traits::Num + Copy + Clone + std::ops::Neg<Output = T>> MaybeSimdProduct
444 for num_complex::Complex<T>
445{
446}
447impl<T: num_traits::Num + Copy + Clone + std::ops::Neg<Output = T>> MaybeSimdSumSquares
448 for num_complex::Complex<T>
449{
450}
451impl<T: num_traits::Num + Copy + Clone + std::ops::Neg<Output = T>> MaybeSimdBlockFold
452 for num_complex::Complex<T>
453{
454}
455
456#[cfg(not(feature = "simd"))]
458impl MaybeSimdOps for f32 {}
459#[cfg(not(feature = "simd"))]
460impl MaybeSimdProduct for f32 {}
461#[cfg(not(feature = "simd"))]
462impl MaybeSimdSumSquares for f32 {}
463
464#[cfg(not(feature = "simd"))]
465impl MaybeSimdOps for f64 {}
466#[cfg(not(feature = "simd"))]
467impl MaybeSimdProduct for f64 {}
468#[cfg(not(feature = "simd"))]
469impl MaybeSimdSumSquares for f64 {}
470#[cfg(not(feature = "simd"))]
471impl MaybeSimdBlockFold for f32 {}
472#[cfg(not(feature = "simd"))]
473impl MaybeSimdBlockFold for f64 {}
474
475#[cfg(feature = "simd")]
476mod simd_impls {
477 use super::{
478 MaybeSimdBlockFold, MaybeSimdOps, MaybeSimdProduct, MaybeSimdSumSquares, BLOCK_FOLD_MIN_LEN,
479 };
480 use pulp::{Simd, WithSimd};
481
482 impl MaybeSimdOps for f32 {
483 fn try_simd_sum(src: &[f32]) -> Option<f32> {
484 struct Sum<'a>(&'a [f32]);
485 impl<'a> WithSimd for Sum<'a> {
486 type Output = f32;
487
488 #[inline(always)]
489 fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
490 let (head, tail) = S::as_simd_f32s(self.0);
491
492 let mut acc0 = simd.splat_f32s(0.0);
493 let mut acc1 = simd.splat_f32s(0.0);
494 let mut acc2 = simd.splat_f32s(0.0);
495 let mut acc3 = simd.splat_f32s(0.0);
496
497 let mut i = 0usize;
498 while i + 4 <= head.len() {
499 acc0 = simd.add_f32s(acc0, head[i]);
500 acc1 = simd.add_f32s(acc1, head[i + 1]);
501 acc2 = simd.add_f32s(acc2, head[i + 2]);
502 acc3 = simd.add_f32s(acc3, head[i + 3]);
503 i += 4;
504 }
505 for &v in &head[i..] {
506 acc0 = simd.add_f32s(acc0, v);
507 }
508
509 let acc = simd.add_f32s(simd.add_f32s(acc0, acc1), simd.add_f32s(acc2, acc3));
510 let mut sum = simd.reduce_sum_f32s(acc);
511 for &x in tail {
512 sum += x;
513 }
514 sum
515 }
516 }
517
518 Some(pulp::Arch::new().dispatch(Sum(src)))
519 }
520
521 fn try_simd_dot(a: &[f32], b: &[f32]) -> Option<f32> {
522 try_simd_dot_f32(a, b)
523 }
524 }
525
526 impl MaybeSimdProduct for f32 {
527 fn try_simd_product(src: &[f32]) -> Option<f32> {
528 struct Product<'a>(&'a [f32]);
529 impl<'a> WithSimd for Product<'a> {
530 type Output = f32;
531
532 #[inline(always)]
533 fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
534 let (head, tail) = S::as_simd_f32s(self.0);
535
536 let mut acc0 = simd.splat_f32s(1.0);
537 let mut acc1 = simd.splat_f32s(1.0);
538 let mut acc2 = simd.splat_f32s(1.0);
539 let mut acc3 = simd.splat_f32s(1.0);
540
541 let mut i = 0usize;
542 while i + 4 <= head.len() {
543 acc0 = simd.mul_f32s(acc0, head[i]);
544 acc1 = simd.mul_f32s(acc1, head[i + 1]);
545 acc2 = simd.mul_f32s(acc2, head[i + 2]);
546 acc3 = simd.mul_f32s(acc3, head[i + 3]);
547 i += 4;
548 }
549 for &value in &head[i..] {
550 acc0 = simd.mul_f32s(acc0, value);
551 }
552
553 let acc = simd.mul_f32s(simd.mul_f32s(acc0, acc1), simd.mul_f32s(acc2, acc3));
554 let mut product = simd.reduce_product_f32s(acc);
555 for &value in tail {
556 product *= value;
557 }
558 product
559 }
560 }
561
562 Some(pulp::Arch::new().dispatch(Product(src)))
563 }
564 }
565
566 impl MaybeSimdSumSquares for f32 {
567 fn try_simd_sum_squares(src: &[f32]) -> Option<f32> {
568 struct SumSquares<'a>(&'a [f32]);
569 impl<'a> WithSimd for SumSquares<'a> {
570 type Output = f32;
571
572 #[inline(always)]
573 fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
574 let (head, tail) = S::as_simd_f32s(self.0);
575 let mut acc0 = simd.splat_f32s(0.0);
576 let mut acc1 = simd.splat_f32s(0.0);
577 let mut acc2 = simd.splat_f32s(0.0);
578 let mut acc3 = simd.splat_f32s(0.0);
579
580 let mut i = 0usize;
581 while i + 4 <= head.len() {
582 let square0 = simd.mul_f32s(head[i], head[i]);
583 let square1 = simd.mul_f32s(head[i + 1], head[i + 1]);
584 let square2 = simd.mul_f32s(head[i + 2], head[i + 2]);
585 let square3 = simd.mul_f32s(head[i + 3], head[i + 3]);
586 acc0 = simd.add_f32s(acc0, square0);
587 acc1 = simd.add_f32s(acc1, square1);
588 acc2 = simd.add_f32s(acc2, square2);
589 acc3 = simd.add_f32s(acc3, square3);
590 i += 4;
591 }
592 for &value in &head[i..] {
593 let square = simd.mul_f32s(value, value);
594 acc0 = simd.add_f32s(acc0, square);
595 }
596
597 let acc = simd.add_f32s(simd.add_f32s(acc0, acc1), simd.add_f32s(acc2, acc3));
598 let mut sum = simd.reduce_sum_f32s(acc);
599 for &value in tail {
600 let square = value * value;
601 sum += square;
602 }
603 sum
604 }
605 }
606
607 Some(pulp::Arch::new().dispatch(SumSquares(src)))
608 }
609 }
610
611 fn try_simd_dot_f32(a: &[f32], b: &[f32]) -> Option<f32> {
612 struct Dot<'a> {
613 a: &'a [f32],
614 b: &'a [f32],
615 }
616 impl<'a> WithSimd for Dot<'a> {
617 type Output = f32;
618
619 #[inline(always)]
620 fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
621 debug_assert_eq!(self.a.len(), self.b.len());
622 let (a_head, a_tail) = S::as_simd_f32s(self.a);
623 let (b_head, b_tail) = S::as_simd_f32s(self.b);
624 debug_assert_eq!(a_head.len(), b_head.len());
625 debug_assert_eq!(a_tail.len(), b_tail.len());
626
627 let mut acc0 = simd.splat_f32s(0.0);
628 let mut acc1 = simd.splat_f32s(0.0);
629 let mut acc2 = simd.splat_f32s(0.0);
630 let mut acc3 = simd.splat_f32s(0.0);
631
632 let mut i = 0usize;
633 while i + 4 <= a_head.len() {
634 acc0 = simd.mul_add_f32s(a_head[i], b_head[i], acc0);
635 acc1 = simd.mul_add_f32s(a_head[i + 1], b_head[i + 1], acc1);
636 acc2 = simd.mul_add_f32s(a_head[i + 2], b_head[i + 2], acc2);
637 acc3 = simd.mul_add_f32s(a_head[i + 3], b_head[i + 3], acc3);
638 i += 4;
639 }
640 for j in i..a_head.len() {
641 acc0 = simd.mul_add_f32s(a_head[j], b_head[j], acc0);
642 }
643
644 let acc = simd.add_f32s(simd.add_f32s(acc0, acc1), simd.add_f32s(acc2, acc3));
645 let mut sum = simd.reduce_sum_f32s(acc);
646 for (&x, &y) in a_tail.iter().zip(b_tail.iter()) {
647 sum += x * y;
648 }
649 sum
650 }
651 }
652
653 Some(pulp::Arch::new().dispatch(Dot { a, b }))
654 }
655
656 impl MaybeSimdBlockFold for f32 {
657 fn try_simd_add_block(acc: &mut [f32], values: &[f32]) -> bool {
658 if acc.len() < BLOCK_FOLD_MIN_LEN {
659 return false;
660 }
661 struct AddBlock<'a, 'b>(&'a mut [f32], &'b [f32]);
662 impl pulp::WithSimd for AddBlock<'_, '_> {
663 type Output = ();
664 #[inline(always)]
665 fn with_simd<S: Simd>(self, simd: S) {
666 let (ah, at) = S::as_mut_simd_f32s(self.0);
667 let (vh, vt) = S::as_simd_f32s(self.1);
668 for (a, &v) in ah.iter_mut().zip(vh) {
671 *a = simd.add_f32s(*a, v);
672 }
673 for (a, &v) in at.iter_mut().zip(vt) {
674 *a += v;
675 }
676 }
677 }
678 pulp::Arch::new().dispatch(AddBlock(acc, values));
679 true
680 }
681
682 fn try_simd_mul_block(acc: &mut [f32], values: &[f32]) -> bool {
683 if acc.len() < BLOCK_FOLD_MIN_LEN {
684 return false;
685 }
686 struct MulBlock<'a, 'b>(&'a mut [f32], &'b [f32]);
687 impl pulp::WithSimd for MulBlock<'_, '_> {
688 type Output = ();
689 #[inline(always)]
690 fn with_simd<S: Simd>(self, simd: S) {
691 let (ah, at) = S::as_mut_simd_f32s(self.0);
692 let (vh, vt) = S::as_simd_f32s(self.1);
693 for (a, &v) in ah.iter_mut().zip(vh) {
694 *a = simd.mul_f32s(*a, v);
695 }
696 for (a, &v) in at.iter_mut().zip(vt) {
697 *a *= v;
698 }
699 }
700 }
701 pulp::Arch::new().dispatch(MulBlock(acc, values));
702 true
703 }
704
705 fn try_simd_sum_squares_block(acc: &mut [f32], values: &[f32]) -> bool {
706 if acc.len() < BLOCK_FOLD_MIN_LEN {
707 return false;
708 }
709 struct SumSquaresBlock<'a, 'b>(&'a mut [f32], &'b [f32]);
710 impl pulp::WithSimd for SumSquaresBlock<'_, '_> {
711 type Output = ();
712 #[inline(always)]
713 fn with_simd<S: Simd>(self, simd: S) {
714 let (ah, at) = S::as_mut_simd_f32s(self.0);
715 let (vh, vt) = S::as_simd_f32s(self.1);
716 for (a, &v) in ah.iter_mut().zip(vh) {
717 let sq = simd.mul_f32s(v, v);
718 *a = simd.add_f32s(*a, sq);
719 }
720 for (a, &v) in at.iter_mut().zip(vt) {
721 *a += v * v;
722 }
723 }
724 }
725 pulp::Arch::new().dispatch(SumSquaresBlock(acc, values));
726 true
727 }
728 }
729
730 impl MaybeSimdOps for f64 {
731 fn try_simd_sum(src: &[f64]) -> Option<f64> {
732 struct Sum<'a>(&'a [f64]);
733 impl<'a> WithSimd for Sum<'a> {
734 type Output = f64;
735
736 #[inline(always)]
737 fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
738 let (head, tail) = S::as_simd_f64s(self.0);
739
740 let mut acc0 = simd.splat_f64s(0.0);
741 let mut acc1 = simd.splat_f64s(0.0);
742 let mut acc2 = simd.splat_f64s(0.0);
743 let mut acc3 = simd.splat_f64s(0.0);
744
745 let mut i = 0usize;
746 while i + 4 <= head.len() {
747 acc0 = simd.add_f64s(acc0, head[i]);
748 acc1 = simd.add_f64s(acc1, head[i + 1]);
749 acc2 = simd.add_f64s(acc2, head[i + 2]);
750 acc3 = simd.add_f64s(acc3, head[i + 3]);
751 i += 4;
752 }
753 for &v in &head[i..] {
754 acc0 = simd.add_f64s(acc0, v);
755 }
756
757 let acc = simd.add_f64s(simd.add_f64s(acc0, acc1), simd.add_f64s(acc2, acc3));
758 let mut sum = simd.reduce_sum_f64s(acc);
759 for &x in tail {
760 sum += x;
761 }
762 sum
763 }
764 }
765
766 Some(pulp::Arch::new().dispatch(Sum(src)))
767 }
768
769 fn try_simd_dot(a: &[f64], b: &[f64]) -> Option<f64> {
770 try_simd_dot_f64(a, b)
771 }
772 }
773
774 impl MaybeSimdProduct for f64 {
775 fn try_simd_product(src: &[f64]) -> Option<f64> {
776 struct Product<'a>(&'a [f64]);
777 impl<'a> WithSimd for Product<'a> {
778 type Output = f64;
779
780 #[inline(always)]
781 fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
782 let (head, tail) = S::as_simd_f64s(self.0);
783
784 let mut acc0 = simd.splat_f64s(1.0);
785 let mut acc1 = simd.splat_f64s(1.0);
786 let mut acc2 = simd.splat_f64s(1.0);
787 let mut acc3 = simd.splat_f64s(1.0);
788
789 let mut i = 0usize;
790 while i + 4 <= head.len() {
791 acc0 = simd.mul_f64s(acc0, head[i]);
792 acc1 = simd.mul_f64s(acc1, head[i + 1]);
793 acc2 = simd.mul_f64s(acc2, head[i + 2]);
794 acc3 = simd.mul_f64s(acc3, head[i + 3]);
795 i += 4;
796 }
797 for &value in &head[i..] {
798 acc0 = simd.mul_f64s(acc0, value);
799 }
800
801 let acc = simd.mul_f64s(simd.mul_f64s(acc0, acc1), simd.mul_f64s(acc2, acc3));
802 let mut product = simd.reduce_product_f64s(acc);
803 for &value in tail {
804 product *= value;
805 }
806 product
807 }
808 }
809
810 Some(pulp::Arch::new().dispatch(Product(src)))
811 }
812 }
813
814 impl MaybeSimdSumSquares for f64 {
815 fn try_simd_sum_squares(src: &[f64]) -> Option<f64> {
816 struct SumSquares<'a>(&'a [f64]);
817 impl<'a> WithSimd for SumSquares<'a> {
818 type Output = f64;
819
820 #[inline(always)]
821 fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
822 let (head, tail) = S::as_simd_f64s(self.0);
823 let mut acc0 = simd.splat_f64s(0.0);
824 let mut acc1 = simd.splat_f64s(0.0);
825 let mut acc2 = simd.splat_f64s(0.0);
826 let mut acc3 = simd.splat_f64s(0.0);
827
828 let mut i = 0usize;
829 while i + 4 <= head.len() {
830 let square0 = simd.mul_f64s(head[i], head[i]);
831 let square1 = simd.mul_f64s(head[i + 1], head[i + 1]);
832 let square2 = simd.mul_f64s(head[i + 2], head[i + 2]);
833 let square3 = simd.mul_f64s(head[i + 3], head[i + 3]);
834 acc0 = simd.add_f64s(acc0, square0);
835 acc1 = simd.add_f64s(acc1, square1);
836 acc2 = simd.add_f64s(acc2, square2);
837 acc3 = simd.add_f64s(acc3, square3);
838 i += 4;
839 }
840 for &value in &head[i..] {
841 let square = simd.mul_f64s(value, value);
842 acc0 = simd.add_f64s(acc0, square);
843 }
844
845 let acc = simd.add_f64s(simd.add_f64s(acc0, acc1), simd.add_f64s(acc2, acc3));
846 let mut sum = simd.reduce_sum_f64s(acc);
847 for &value in tail {
848 let square = value * value;
849 sum += square;
850 }
851 sum
852 }
853 }
854
855 Some(pulp::Arch::new().dispatch(SumSquares(src)))
856 }
857 }
858
859 fn try_simd_dot_f64(a: &[f64], b: &[f64]) -> Option<f64> {
860 struct Dot<'a> {
861 a: &'a [f64],
862 b: &'a [f64],
863 }
864 impl<'a> WithSimd for Dot<'a> {
865 type Output = f64;
866
867 #[inline(always)]
868 fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
869 debug_assert_eq!(self.a.len(), self.b.len());
870 let (a_head, a_tail) = S::as_simd_f64s(self.a);
871 let (b_head, b_tail) = S::as_simd_f64s(self.b);
872 debug_assert_eq!(a_head.len(), b_head.len());
873 debug_assert_eq!(a_tail.len(), b_tail.len());
874
875 let mut acc0 = simd.splat_f64s(0.0);
876 let mut acc1 = simd.splat_f64s(0.0);
877 let mut acc2 = simd.splat_f64s(0.0);
878 let mut acc3 = simd.splat_f64s(0.0);
879
880 let mut i = 0usize;
881 while i + 4 <= a_head.len() {
882 acc0 = simd.mul_add_f64s(a_head[i], b_head[i], acc0);
883 acc1 = simd.mul_add_f64s(a_head[i + 1], b_head[i + 1], acc1);
884 acc2 = simd.mul_add_f64s(a_head[i + 2], b_head[i + 2], acc2);
885 acc3 = simd.mul_add_f64s(a_head[i + 3], b_head[i + 3], acc3);
886 i += 4;
887 }
888 for j in i..a_head.len() {
889 acc0 = simd.mul_add_f64s(a_head[j], b_head[j], acc0);
890 }
891
892 let acc = simd.add_f64s(simd.add_f64s(acc0, acc1), simd.add_f64s(acc2, acc3));
893 let mut sum = simd.reduce_sum_f64s(acc);
894 for (&x, &y) in a_tail.iter().zip(b_tail.iter()) {
895 sum += x * y;
896 }
897 sum
898 }
899 }
900
901 Some(pulp::Arch::new().dispatch(Dot { a, b }))
902 }
903
904 impl MaybeSimdBlockFold for f64 {
905 fn try_simd_add_block(acc: &mut [f64], values: &[f64]) -> bool {
906 if acc.len() < BLOCK_FOLD_MIN_LEN {
907 return false;
908 }
909 struct AddBlock<'a, 'b>(&'a mut [f64], &'b [f64]);
910 impl pulp::WithSimd for AddBlock<'_, '_> {
911 type Output = ();
912 #[inline(always)]
913 fn with_simd<S: Simd>(self, simd: S) {
914 let (ah, at) = S::as_mut_simd_f64s(self.0);
915 let (vh, vt) = S::as_simd_f64s(self.1);
916 for (a, &v) in ah.iter_mut().zip(vh) {
919 *a = simd.add_f64s(*a, v);
920 }
921 for (a, &v) in at.iter_mut().zip(vt) {
922 *a += v;
923 }
924 }
925 }
926 pulp::Arch::new().dispatch(AddBlock(acc, values));
927 true
928 }
929
930 fn try_simd_mul_block(acc: &mut [f64], values: &[f64]) -> bool {
931 if acc.len() < BLOCK_FOLD_MIN_LEN {
932 return false;
933 }
934 struct MulBlock<'a, 'b>(&'a mut [f64], &'b [f64]);
935 impl pulp::WithSimd for MulBlock<'_, '_> {
936 type Output = ();
937 #[inline(always)]
938 fn with_simd<S: Simd>(self, simd: S) {
939 let (ah, at) = S::as_mut_simd_f64s(self.0);
940 let (vh, vt) = S::as_simd_f64s(self.1);
941 for (a, &v) in ah.iter_mut().zip(vh) {
942 *a = simd.mul_f64s(*a, v);
943 }
944 for (a, &v) in at.iter_mut().zip(vt) {
945 *a *= v;
946 }
947 }
948 }
949 pulp::Arch::new().dispatch(MulBlock(acc, values));
950 true
951 }
952
953 fn try_simd_sum_squares_block(acc: &mut [f64], values: &[f64]) -> bool {
954 if acc.len() < BLOCK_FOLD_MIN_LEN {
955 return false;
956 }
957 struct SumSquaresBlock<'a, 'b>(&'a mut [f64], &'b [f64]);
958 impl pulp::WithSimd for SumSquaresBlock<'_, '_> {
959 type Output = ();
960 #[inline(always)]
961 fn with_simd<S: Simd>(self, simd: S) {
962 let (ah, at) = S::as_mut_simd_f64s(self.0);
963 let (vh, vt) = S::as_simd_f64s(self.1);
964 for (a, &v) in ah.iter_mut().zip(vh) {
965 let sq = simd.mul_f64s(v, v);
966 *a = simd.add_f64s(*a, sq);
967 }
968 for (a, &v) in at.iter_mut().zip(vt) {
969 *a += v * v;
970 }
971 }
972 }
973 pulp::Arch::new().dispatch(SumSquaresBlock(acc, values));
974 true
975 }
976 }
977}
978
979#[cfg(test)]
980#[path = "simd/tests/tests.rs"]
981mod tests;