Skip to main content

strided_basic/
simd.rs

1#[inline(always)]
2pub(crate) fn dispatch<R>(f: impl FnOnce() -> R) -> R {
3    #[cfg(feature = "simd")]
4    {
5        pulp::Arch::new().dispatch(f)
6    }
7    #[cfg(not(feature = "simd"))]
8    {
9        f()
10    }
11}
12
13#[inline(always)]
14pub(crate) fn dispatch_if_large<R>(len: usize, f: impl FnOnce() -> R) -> R {
15    // Avoid runtime-dispatch overhead for tiny loops (especially common for small-array cases).
16    // This is a heuristic; correctness does not depend on it.
17    if len >= 64 {
18        dispatch(f)
19    } else {
20        f()
21    }
22}
23
24#[cfg(feature = "simd")]
25#[inline(always)]
26unsafe fn cast_slice<T, U>(src: &[T]) -> &[U] {
27    debug_assert_eq!(std::mem::size_of::<T>(), std::mem::size_of::<U>());
28    unsafe { std::slice::from_raw_parts(src.as_ptr().cast::<U>(), src.len()) }
29}
30
31#[cfg(feature = "simd")]
32macro_rules! impl_simd_mul_ptr {
33    (
34        $mul_into:ident,
35        $ty:ty,
36        $split:ident,
37        $split_uninit:ident,
38        $load:ident,
39        $mask:ident,
40        $store_ptr:ident,
41        $mul:ident,
42        $mask_scale:expr
43    ) => {
44        unsafe fn $mul_into(dst: *mut $ty, len: usize, a: &[$ty], b: &[$ty]) {
45            struct Mul<'a> {
46                dst: *mut $ty,
47                len: usize,
48                a: &'a [$ty],
49                b: &'a [$ty],
50            }
51
52            impl<'a> pulp::WithSimd for Mul<'a> {
53                type Output = ();
54
55                #[inline(always)]
56                fn with_simd<S: pulp::Simd>(self, simd: S) -> Self::Output {
57                    debug_assert_eq!(self.len, self.a.len());
58                    debug_assert_eq!(self.len, self.b.len());
59
60                    // SAFETY: the caller supplies len writable, nonaliasing
61                    // elements. MaybeUninit does not assert initialized output.
62                    let output = unsafe {
63                        core::slice::from_raw_parts_mut(
64                            self.dst.cast::<core::mem::MaybeUninit<$ty>>(),
65                            self.len,
66                        )
67                    };
68                    let (dest, tail) = S::$split_uninit(output);
69                    let (a, a_tail) = S::$split(self.a);
70                    let (b, b_tail) = S::$split(self.b);
71                    // Full vectors use ordinary stores. Pulp's partial-access
72                    // machinery is needed only for the final incomplete vector.
73                    for ((dst, &a), &b) in dest.iter_mut().zip(a).zip(b) {
74                        dst.write(simd.$mul(a, b));
75                    }
76                    if !tail.is_empty() {
77                        let va = simd.$load(a_tail);
78                        let vb = simd.$load(b_tail);
79                        // SAFETY: the mask writes exactly the remaining lanes.
80                        unsafe {
81                            simd.$store_ptr(
82                                simd.$mask(0, (tail.len() * $mask_scale) as _),
83                                tail.as_mut_ptr().cast::<$ty>(),
84                                simd.$mul(va, vb),
85                            );
86                        }
87                    }
88                }
89            }
90
91            pulp::Arch::new().dispatch(Mul { dst, len, a, b });
92        }
93    };
94}
95
96#[cfg(feature = "simd")]
97impl_simd_mul_ptr!(
98    simd_mul_f32_into,
99    f32,
100    as_simd_f32s,
101    as_uninit_mut_simd_f32s,
102    partial_load_f32s,
103    mask_between_m32s,
104    mask_store_ptr_f32s,
105    mul_f32s,
106    1
107);
108
109#[cfg(feature = "simd")]
110impl_simd_mul_ptr!(
111    simd_mul_f64_into,
112    f64,
113    as_simd_f64s,
114    as_uninit_mut_simd_f64s,
115    partial_load_f64s,
116    mask_between_m64s,
117    mask_store_ptr_f64s,
118    mul_f64s,
119    1
120);
121
122#[cfg(feature = "simd")]
123impl_simd_mul_ptr!(
124    simd_mul_c32_into,
125    num_complex::Complex32,
126    as_simd_c32s,
127    as_uninit_mut_simd_c32s,
128    partial_load_c32s,
129    mask_between_m32s,
130    mask_store_ptr_c32s,
131    mul_e_c32s,
132    2
133);
134
135#[cfg(feature = "simd")]
136impl_simd_mul_ptr!(
137    simd_mul_c64_into,
138    num_complex::Complex64,
139    as_simd_c64s,
140    as_uninit_mut_simd_c64s,
141    partial_load_c64s,
142    mask_between_m64s,
143    mask_store_ptr_c64s,
144    mul_e_c64s,
145    2
146);
147
148#[cfg(feature = "simd")]
149#[inline]
150// Only `simd/tests` call this safe wrapper; production calls `try_mul_contiguous_ptr`.
151#[cfg_attr(not(test), allow(dead_code))]
152pub(crate) fn try_mul_contiguous<D: 'static, A: 'static, B: 'static>(
153    dst: &mut [D],
154    a: &[A],
155    b: &[B],
156) -> bool {
157    unsafe { try_mul_contiguous_ptr(dst.as_mut_ptr(), dst.len(), a, b) }
158}
159
160#[cfg(feature = "simd")]
161#[inline]
162pub(crate) unsafe fn try_mul_contiguous_ptr<D: 'static, A: 'static, B: 'static>(
163    dst: *mut D,
164    len: usize,
165    a: &[A],
166    b: &[B],
167) -> bool {
168    use std::any::TypeId;
169
170    macro_rules! try_same_type {
171        ($ty:ty, $mul_into:ident) => {
172            if TypeId::of::<D>() == TypeId::of::<$ty>()
173                && TypeId::of::<A>() == TypeId::of::<$ty>()
174                && TypeId::of::<B>() == TypeId::of::<$ty>()
175            {
176                unsafe { $mul_into(dst.cast::<$ty>(), len, cast_slice(a), cast_slice(b)) };
177                return true;
178            }
179        };
180    }
181
182    try_same_type!(f64, simd_mul_f64_into);
183    try_same_type!(f32, simd_mul_f32_into);
184    try_same_type!(num_complex::Complex64, simd_mul_c64_into);
185    try_same_type!(num_complex::Complex32, simd_mul_c32_into);
186
187    false
188}
189
190#[cfg(not(feature = "simd"))]
191#[inline]
192// Same as the `simd` branch above: only the tests call this safe wrapper.
193#[cfg_attr(not(test), allow(dead_code))]
194pub(crate) fn try_mul_contiguous<D: 'static, A: 'static, B: 'static>(
195    _dst: &mut [D],
196    _a: &[A],
197    _b: &[B],
198) -> bool {
199    false
200}
201
202#[cfg(not(feature = "simd"))]
203#[inline]
204pub(crate) unsafe fn try_mul_contiguous_ptr<D: 'static, A: 'static, B: 'static>(
205    _dst: *mut D,
206    _len: usize,
207    _a: &[A],
208    _b: &[B],
209) -> bool {
210    false
211}
212
213const TRANSPOSE_TILE: usize = 8;
214
215#[inline(always)]
216unsafe fn mul_transposed_scalar_rhs_source_contiguous<T>(
217    dst: *mut T,
218    src: *const T,
219    scalar: T,
220    inner_len: usize,
221    row_len: usize,
222    src_fast_stride: isize,
223    src_row_stride: isize,
224) where
225    T: Copy + std::ops::Mul<Output = T>,
226{
227    let mut inner0 = 0usize;
228
229    while inner0 < inner_len {
230        let inner_count = TRANSPOSE_TILE.min(inner_len - inner0);
231        let mut row0 = 0usize;
232
233        while row0 < row_len {
234            let row_count = TRANSPOSE_TILE.min(row_len - row0);
235
236            for inner in 0..inner_count {
237                let src_base = src.offset((inner0 + inner) as isize * src_fast_stride);
238                for row in 0..row_count {
239                    let row_index = row0 + row;
240                    let src_offset = row_index as isize * src_row_stride;
241                    *dst.add(row_index * inner_len + inner0 + inner) =
242                        *src_base.offset(src_offset) * scalar;
243                }
244            }
245
246            row0 += TRANSPOSE_TILE;
247        }
248
249        inner0 += TRANSPOSE_TILE;
250    }
251}
252
253#[inline(always)]
254unsafe fn mul_transposed_scalar_lhs_source_contiguous<T>(
255    dst: *mut T,
256    scalar: T,
257    src: *const T,
258    inner_len: usize,
259    row_len: usize,
260    src_fast_stride: isize,
261    src_row_stride: isize,
262) where
263    T: Copy + std::ops::Mul<Output = T>,
264{
265    let mut inner0 = 0usize;
266
267    while inner0 < inner_len {
268        let inner_count = TRANSPOSE_TILE.min(inner_len - inner0);
269        let mut row0 = 0usize;
270
271        while row0 < row_len {
272            let row_count = TRANSPOSE_TILE.min(row_len - row0);
273
274            for inner in 0..inner_count {
275                let src_base = src.offset((inner0 + inner) as isize * src_fast_stride);
276                for row in 0..row_count {
277                    let row_index = row0 + row;
278                    let src_offset = row_index as isize * src_row_stride;
279                    *dst.add(row_index * inner_len + inner0 + inner) =
280                        scalar * *src_base.offset(src_offset);
281                }
282            }
283
284            row0 += TRANSPOSE_TILE;
285        }
286
287        inner0 += TRANSPOSE_TILE;
288    }
289}
290
291#[inline]
292pub(crate) unsafe fn try_mul_transposed_scalar_rhs_2d<D: 'static, A: 'static, B: 'static>(
293    dst: *mut D,
294    src: *const A,
295    scalar: *const B,
296    inner_len: usize,
297    row_len: usize,
298    src_fast_stride: isize,
299    src_row_stride: isize,
300) -> bool {
301    use std::any::TypeId;
302
303    macro_rules! try_same_type {
304        ($ty:ty) => {
305            if TypeId::of::<D>() == TypeId::of::<$ty>()
306                && TypeId::of::<A>() == TypeId::of::<$ty>()
307                && TypeId::of::<B>() == TypeId::of::<$ty>()
308            {
309                // One runtime target-feature dispatch per tile sweep (tensor4all/strided-rs#340).
310                dispatch_if_large(inner_len * row_len, || unsafe {
311                    mul_transposed_scalar_rhs_source_contiguous(
312                        dst.cast::<$ty>(),
313                        src.cast::<$ty>(),
314                        *scalar.cast::<$ty>(),
315                        inner_len,
316                        row_len,
317                        src_fast_stride,
318                        src_row_stride,
319                    );
320                });
321                return true;
322            }
323        };
324    }
325
326    try_same_type!(f64);
327    try_same_type!(f32);
328    try_same_type!(num_complex::Complex64);
329    try_same_type!(num_complex::Complex32);
330
331    false
332}
333
334#[inline]
335pub(crate) unsafe fn try_mul_transposed_scalar_lhs_2d<D: 'static, A: 'static, B: 'static>(
336    dst: *mut D,
337    scalar: *const A,
338    src: *const B,
339    inner_len: usize,
340    row_len: usize,
341    src_fast_stride: isize,
342    src_row_stride: isize,
343) -> bool {
344    use std::any::TypeId;
345
346    macro_rules! try_same_type {
347        ($ty:ty) => {
348            if TypeId::of::<D>() == TypeId::of::<$ty>()
349                && TypeId::of::<A>() == TypeId::of::<$ty>()
350                && TypeId::of::<B>() == TypeId::of::<$ty>()
351            {
352                // One runtime target-feature dispatch per tile sweep (tensor4all/strided-rs#340).
353                dispatch_if_large(inner_len * row_len, || unsafe {
354                    mul_transposed_scalar_lhs_source_contiguous(
355                        dst.cast::<$ty>(),
356                        *scalar.cast::<$ty>(),
357                        src.cast::<$ty>(),
358                        inner_len,
359                        row_len,
360                        src_fast_stride,
361                        src_row_stride,
362                    );
363                });
364                return true;
365            }
366        };
367    }
368
369    try_same_type!(f64);
370    try_same_type!(f32);
371    try_same_type!(num_complex::Complex64);
372    try_same_type!(num_complex::Complex32);
373
374    false
375}
376
377/// Trait for types that may have SIMD-accelerated sum/dot operations.
378///
379/// Default implementations return `None` (no SIMD available).
380/// f32/f64 override these with SIMD kernels when the `simd` feature is enabled.
381pub trait MaybeSimdOps: Copy + Sized {
382    fn try_simd_sum(_src: &[Self]) -> Option<Self> {
383        None
384    }
385    fn try_simd_dot(_a: &[Self], _b: &[Self]) -> Option<Self> {
386        None
387    }
388}
389
390pub(crate) trait MaybeSimdProduct: Copy + Sized {
391    fn try_simd_product(_src: &[Self]) -> Option<Self> {
392        None
393    }
394}
395
396pub(crate) trait MaybeSimdSumSquares: Copy + Sized {
397    fn try_simd_sum_squares(_src: &[Self]) -> Option<Self> {
398        None
399    }
400}
401
402/// Minimum accumulator-block length worth a SIMD dispatch.
403#[cfg(feature = "simd")]
404const BLOCK_FOLD_MIN_LEN: usize = 16;
405
406/// SIMD fold of one unit-stride value run into adjacent accumulators.
407///
408/// This is a reduction-kernel implementation hook, not part of the public
409/// surface. Every method requires `acc.len() == values.len()`; the reduction
410/// callers always pass the same block length for both, so the vector loop and
411/// the scalar tail cover exactly the same indices. A method returns false when
412/// no SIMD path applies and leaves `acc` untouched.
413pub(crate) trait MaybeSimdBlockFold: Copy + Sized {
414    fn try_simd_add_block(_acc: &mut [Self], _values: &[Self]) -> bool {
415        false
416    }
417    fn try_simd_mul_block(_acc: &mut [Self], _values: &[Self]) -> bool {
418        false
419    }
420    fn try_simd_sum_squares_block(_acc: &mut [Self], _values: &[Self]) -> bool {
421        false
422    }
423}
424
425// Default (no-op) impls for integer types and Complex
426macro_rules! impl_no_simd {
427    ($($t:ty),*) => {
428        $(
429            impl MaybeSimdOps for $t {}
430            impl MaybeSimdProduct for $t {}
431            impl MaybeSimdSumSquares for $t {}
432            impl MaybeSimdBlockFold for $t {}
433        )*
434    };
435}
436
437impl_no_simd!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize);
438
439impl<T: num_traits::Num + Copy + Clone + std::ops::Neg<Output = T>> MaybeSimdOps
440    for num_complex::Complex<T>
441{
442}
443impl<T: num_traits::Num + Copy + Clone + std::ops::Neg<Output = T>> MaybeSimdProduct
444    for num_complex::Complex<T>
445{
446}
447impl<T: num_traits::Num + Copy + Clone + std::ops::Neg<Output = T>> MaybeSimdSumSquares
448    for num_complex::Complex<T>
449{
450}
451impl<T: num_traits::Num + Copy + Clone + std::ops::Neg<Output = T>> MaybeSimdBlockFold
452    for num_complex::Complex<T>
453{
454}
455
456// f32/f64: SIMD-accelerated when feature enabled, no-op otherwise
457#[cfg(not(feature = "simd"))]
458impl MaybeSimdOps for f32 {}
459#[cfg(not(feature = "simd"))]
460impl MaybeSimdProduct for f32 {}
461#[cfg(not(feature = "simd"))]
462impl MaybeSimdSumSquares for f32 {}
463
464#[cfg(not(feature = "simd"))]
465impl MaybeSimdOps for f64 {}
466#[cfg(not(feature = "simd"))]
467impl MaybeSimdProduct for f64 {}
468#[cfg(not(feature = "simd"))]
469impl MaybeSimdSumSquares for f64 {}
470#[cfg(not(feature = "simd"))]
471impl MaybeSimdBlockFold for f32 {}
472#[cfg(not(feature = "simd"))]
473impl MaybeSimdBlockFold for f64 {}
474
475#[cfg(feature = "simd")]
476mod simd_impls {
477    use super::{
478        MaybeSimdBlockFold, MaybeSimdOps, MaybeSimdProduct, MaybeSimdSumSquares, BLOCK_FOLD_MIN_LEN,
479    };
480    use pulp::{Simd, WithSimd};
481
482    impl MaybeSimdOps for f32 {
483        fn try_simd_sum(src: &[f32]) -> Option<f32> {
484            struct Sum<'a>(&'a [f32]);
485            impl<'a> WithSimd for Sum<'a> {
486                type Output = f32;
487
488                #[inline(always)]
489                fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
490                    let (head, tail) = S::as_simd_f32s(self.0);
491
492                    let mut acc0 = simd.splat_f32s(0.0);
493                    let mut acc1 = simd.splat_f32s(0.0);
494                    let mut acc2 = simd.splat_f32s(0.0);
495                    let mut acc3 = simd.splat_f32s(0.0);
496
497                    let mut i = 0usize;
498                    while i + 4 <= head.len() {
499                        acc0 = simd.add_f32s(acc0, head[i]);
500                        acc1 = simd.add_f32s(acc1, head[i + 1]);
501                        acc2 = simd.add_f32s(acc2, head[i + 2]);
502                        acc3 = simd.add_f32s(acc3, head[i + 3]);
503                        i += 4;
504                    }
505                    for &v in &head[i..] {
506                        acc0 = simd.add_f32s(acc0, v);
507                    }
508
509                    let acc = simd.add_f32s(simd.add_f32s(acc0, acc1), simd.add_f32s(acc2, acc3));
510                    let mut sum = simd.reduce_sum_f32s(acc);
511                    for &x in tail {
512                        sum += x;
513                    }
514                    sum
515                }
516            }
517
518            Some(pulp::Arch::new().dispatch(Sum(src)))
519        }
520
521        fn try_simd_dot(a: &[f32], b: &[f32]) -> Option<f32> {
522            try_simd_dot_f32(a, b)
523        }
524    }
525
526    impl MaybeSimdProduct for f32 {
527        fn try_simd_product(src: &[f32]) -> Option<f32> {
528            struct Product<'a>(&'a [f32]);
529            impl<'a> WithSimd for Product<'a> {
530                type Output = f32;
531
532                #[inline(always)]
533                fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
534                    let (head, tail) = S::as_simd_f32s(self.0);
535
536                    let mut acc0 = simd.splat_f32s(1.0);
537                    let mut acc1 = simd.splat_f32s(1.0);
538                    let mut acc2 = simd.splat_f32s(1.0);
539                    let mut acc3 = simd.splat_f32s(1.0);
540
541                    let mut i = 0usize;
542                    while i + 4 <= head.len() {
543                        acc0 = simd.mul_f32s(acc0, head[i]);
544                        acc1 = simd.mul_f32s(acc1, head[i + 1]);
545                        acc2 = simd.mul_f32s(acc2, head[i + 2]);
546                        acc3 = simd.mul_f32s(acc3, head[i + 3]);
547                        i += 4;
548                    }
549                    for &value in &head[i..] {
550                        acc0 = simd.mul_f32s(acc0, value);
551                    }
552
553                    let acc = simd.mul_f32s(simd.mul_f32s(acc0, acc1), simd.mul_f32s(acc2, acc3));
554                    let mut product = simd.reduce_product_f32s(acc);
555                    for &value in tail {
556                        product *= value;
557                    }
558                    product
559                }
560            }
561
562            Some(pulp::Arch::new().dispatch(Product(src)))
563        }
564    }
565
566    impl MaybeSimdSumSquares for f32 {
567        fn try_simd_sum_squares(src: &[f32]) -> Option<f32> {
568            struct SumSquares<'a>(&'a [f32]);
569            impl<'a> WithSimd for SumSquares<'a> {
570                type Output = f32;
571
572                #[inline(always)]
573                fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
574                    let (head, tail) = S::as_simd_f32s(self.0);
575                    let mut acc0 = simd.splat_f32s(0.0);
576                    let mut acc1 = simd.splat_f32s(0.0);
577                    let mut acc2 = simd.splat_f32s(0.0);
578                    let mut acc3 = simd.splat_f32s(0.0);
579
580                    let mut i = 0usize;
581                    while i + 4 <= head.len() {
582                        let square0 = simd.mul_f32s(head[i], head[i]);
583                        let square1 = simd.mul_f32s(head[i + 1], head[i + 1]);
584                        let square2 = simd.mul_f32s(head[i + 2], head[i + 2]);
585                        let square3 = simd.mul_f32s(head[i + 3], head[i + 3]);
586                        acc0 = simd.add_f32s(acc0, square0);
587                        acc1 = simd.add_f32s(acc1, square1);
588                        acc2 = simd.add_f32s(acc2, square2);
589                        acc3 = simd.add_f32s(acc3, square3);
590                        i += 4;
591                    }
592                    for &value in &head[i..] {
593                        let square = simd.mul_f32s(value, value);
594                        acc0 = simd.add_f32s(acc0, square);
595                    }
596
597                    let acc = simd.add_f32s(simd.add_f32s(acc0, acc1), simd.add_f32s(acc2, acc3));
598                    let mut sum = simd.reduce_sum_f32s(acc);
599                    for &value in tail {
600                        let square = value * value;
601                        sum += square;
602                    }
603                    sum
604                }
605            }
606
607            Some(pulp::Arch::new().dispatch(SumSquares(src)))
608        }
609    }
610
611    fn try_simd_dot_f32(a: &[f32], b: &[f32]) -> Option<f32> {
612        struct Dot<'a> {
613            a: &'a [f32],
614            b: &'a [f32],
615        }
616        impl<'a> WithSimd for Dot<'a> {
617            type Output = f32;
618
619            #[inline(always)]
620            fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
621                debug_assert_eq!(self.a.len(), self.b.len());
622                let (a_head, a_tail) = S::as_simd_f32s(self.a);
623                let (b_head, b_tail) = S::as_simd_f32s(self.b);
624                debug_assert_eq!(a_head.len(), b_head.len());
625                debug_assert_eq!(a_tail.len(), b_tail.len());
626
627                let mut acc0 = simd.splat_f32s(0.0);
628                let mut acc1 = simd.splat_f32s(0.0);
629                let mut acc2 = simd.splat_f32s(0.0);
630                let mut acc3 = simd.splat_f32s(0.0);
631
632                let mut i = 0usize;
633                while i + 4 <= a_head.len() {
634                    acc0 = simd.mul_add_f32s(a_head[i], b_head[i], acc0);
635                    acc1 = simd.mul_add_f32s(a_head[i + 1], b_head[i + 1], acc1);
636                    acc2 = simd.mul_add_f32s(a_head[i + 2], b_head[i + 2], acc2);
637                    acc3 = simd.mul_add_f32s(a_head[i + 3], b_head[i + 3], acc3);
638                    i += 4;
639                }
640                for j in i..a_head.len() {
641                    acc0 = simd.mul_add_f32s(a_head[j], b_head[j], acc0);
642                }
643
644                let acc = simd.add_f32s(simd.add_f32s(acc0, acc1), simd.add_f32s(acc2, acc3));
645                let mut sum = simd.reduce_sum_f32s(acc);
646                for (&x, &y) in a_tail.iter().zip(b_tail.iter()) {
647                    sum += x * y;
648                }
649                sum
650            }
651        }
652
653        Some(pulp::Arch::new().dispatch(Dot { a, b }))
654    }
655
656    impl MaybeSimdBlockFold for f32 {
657        fn try_simd_add_block(acc: &mut [f32], values: &[f32]) -> bool {
658            if acc.len() < BLOCK_FOLD_MIN_LEN {
659                return false;
660            }
661            struct AddBlock<'a, 'b>(&'a mut [f32], &'b [f32]);
662            impl pulp::WithSimd for AddBlock<'_, '_> {
663                type Output = ();
664                #[inline(always)]
665                fn with_simd<S: Simd>(self, simd: S) {
666                    let (ah, at) = S::as_mut_simd_f32s(self.0);
667                    let (vh, vt) = S::as_simd_f32s(self.1);
668                    // Equal block lengths are the trait contract, so the vector
669                    // prefix and the scalar tail cover the same indices.
670                    for (a, &v) in ah.iter_mut().zip(vh) {
671                        *a = simd.add_f32s(*a, v);
672                    }
673                    for (a, &v) in at.iter_mut().zip(vt) {
674                        *a += v;
675                    }
676                }
677            }
678            pulp::Arch::new().dispatch(AddBlock(acc, values));
679            true
680        }
681
682        fn try_simd_mul_block(acc: &mut [f32], values: &[f32]) -> bool {
683            if acc.len() < BLOCK_FOLD_MIN_LEN {
684                return false;
685            }
686            struct MulBlock<'a, 'b>(&'a mut [f32], &'b [f32]);
687            impl pulp::WithSimd for MulBlock<'_, '_> {
688                type Output = ();
689                #[inline(always)]
690                fn with_simd<S: Simd>(self, simd: S) {
691                    let (ah, at) = S::as_mut_simd_f32s(self.0);
692                    let (vh, vt) = S::as_simd_f32s(self.1);
693                    for (a, &v) in ah.iter_mut().zip(vh) {
694                        *a = simd.mul_f32s(*a, v);
695                    }
696                    for (a, &v) in at.iter_mut().zip(vt) {
697                        *a *= v;
698                    }
699                }
700            }
701            pulp::Arch::new().dispatch(MulBlock(acc, values));
702            true
703        }
704
705        fn try_simd_sum_squares_block(acc: &mut [f32], values: &[f32]) -> bool {
706            if acc.len() < BLOCK_FOLD_MIN_LEN {
707                return false;
708            }
709            struct SumSquaresBlock<'a, 'b>(&'a mut [f32], &'b [f32]);
710            impl pulp::WithSimd for SumSquaresBlock<'_, '_> {
711                type Output = ();
712                #[inline(always)]
713                fn with_simd<S: Simd>(self, simd: S) {
714                    let (ah, at) = S::as_mut_simd_f32s(self.0);
715                    let (vh, vt) = S::as_simd_f32s(self.1);
716                    for (a, &v) in ah.iter_mut().zip(vh) {
717                        let sq = simd.mul_f32s(v, v);
718                        *a = simd.add_f32s(*a, sq);
719                    }
720                    for (a, &v) in at.iter_mut().zip(vt) {
721                        *a += v * v;
722                    }
723                }
724            }
725            pulp::Arch::new().dispatch(SumSquaresBlock(acc, values));
726            true
727        }
728    }
729
730    impl MaybeSimdOps for f64 {
731        fn try_simd_sum(src: &[f64]) -> Option<f64> {
732            struct Sum<'a>(&'a [f64]);
733            impl<'a> WithSimd for Sum<'a> {
734                type Output = f64;
735
736                #[inline(always)]
737                fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
738                    let (head, tail) = S::as_simd_f64s(self.0);
739
740                    let mut acc0 = simd.splat_f64s(0.0);
741                    let mut acc1 = simd.splat_f64s(0.0);
742                    let mut acc2 = simd.splat_f64s(0.0);
743                    let mut acc3 = simd.splat_f64s(0.0);
744
745                    let mut i = 0usize;
746                    while i + 4 <= head.len() {
747                        acc0 = simd.add_f64s(acc0, head[i]);
748                        acc1 = simd.add_f64s(acc1, head[i + 1]);
749                        acc2 = simd.add_f64s(acc2, head[i + 2]);
750                        acc3 = simd.add_f64s(acc3, head[i + 3]);
751                        i += 4;
752                    }
753                    for &v in &head[i..] {
754                        acc0 = simd.add_f64s(acc0, v);
755                    }
756
757                    let acc = simd.add_f64s(simd.add_f64s(acc0, acc1), simd.add_f64s(acc2, acc3));
758                    let mut sum = simd.reduce_sum_f64s(acc);
759                    for &x in tail {
760                        sum += x;
761                    }
762                    sum
763                }
764            }
765
766            Some(pulp::Arch::new().dispatch(Sum(src)))
767        }
768
769        fn try_simd_dot(a: &[f64], b: &[f64]) -> Option<f64> {
770            try_simd_dot_f64(a, b)
771        }
772    }
773
774    impl MaybeSimdProduct for f64 {
775        fn try_simd_product(src: &[f64]) -> Option<f64> {
776            struct Product<'a>(&'a [f64]);
777            impl<'a> WithSimd for Product<'a> {
778                type Output = f64;
779
780                #[inline(always)]
781                fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
782                    let (head, tail) = S::as_simd_f64s(self.0);
783
784                    let mut acc0 = simd.splat_f64s(1.0);
785                    let mut acc1 = simd.splat_f64s(1.0);
786                    let mut acc2 = simd.splat_f64s(1.0);
787                    let mut acc3 = simd.splat_f64s(1.0);
788
789                    let mut i = 0usize;
790                    while i + 4 <= head.len() {
791                        acc0 = simd.mul_f64s(acc0, head[i]);
792                        acc1 = simd.mul_f64s(acc1, head[i + 1]);
793                        acc2 = simd.mul_f64s(acc2, head[i + 2]);
794                        acc3 = simd.mul_f64s(acc3, head[i + 3]);
795                        i += 4;
796                    }
797                    for &value in &head[i..] {
798                        acc0 = simd.mul_f64s(acc0, value);
799                    }
800
801                    let acc = simd.mul_f64s(simd.mul_f64s(acc0, acc1), simd.mul_f64s(acc2, acc3));
802                    let mut product = simd.reduce_product_f64s(acc);
803                    for &value in tail {
804                        product *= value;
805                    }
806                    product
807                }
808            }
809
810            Some(pulp::Arch::new().dispatch(Product(src)))
811        }
812    }
813
814    impl MaybeSimdSumSquares for f64 {
815        fn try_simd_sum_squares(src: &[f64]) -> Option<f64> {
816            struct SumSquares<'a>(&'a [f64]);
817            impl<'a> WithSimd for SumSquares<'a> {
818                type Output = f64;
819
820                #[inline(always)]
821                fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
822                    let (head, tail) = S::as_simd_f64s(self.0);
823                    let mut acc0 = simd.splat_f64s(0.0);
824                    let mut acc1 = simd.splat_f64s(0.0);
825                    let mut acc2 = simd.splat_f64s(0.0);
826                    let mut acc3 = simd.splat_f64s(0.0);
827
828                    let mut i = 0usize;
829                    while i + 4 <= head.len() {
830                        let square0 = simd.mul_f64s(head[i], head[i]);
831                        let square1 = simd.mul_f64s(head[i + 1], head[i + 1]);
832                        let square2 = simd.mul_f64s(head[i + 2], head[i + 2]);
833                        let square3 = simd.mul_f64s(head[i + 3], head[i + 3]);
834                        acc0 = simd.add_f64s(acc0, square0);
835                        acc1 = simd.add_f64s(acc1, square1);
836                        acc2 = simd.add_f64s(acc2, square2);
837                        acc3 = simd.add_f64s(acc3, square3);
838                        i += 4;
839                    }
840                    for &value in &head[i..] {
841                        let square = simd.mul_f64s(value, value);
842                        acc0 = simd.add_f64s(acc0, square);
843                    }
844
845                    let acc = simd.add_f64s(simd.add_f64s(acc0, acc1), simd.add_f64s(acc2, acc3));
846                    let mut sum = simd.reduce_sum_f64s(acc);
847                    for &value in tail {
848                        let square = value * value;
849                        sum += square;
850                    }
851                    sum
852                }
853            }
854
855            Some(pulp::Arch::new().dispatch(SumSquares(src)))
856        }
857    }
858
859    fn try_simd_dot_f64(a: &[f64], b: &[f64]) -> Option<f64> {
860        struct Dot<'a> {
861            a: &'a [f64],
862            b: &'a [f64],
863        }
864        impl<'a> WithSimd for Dot<'a> {
865            type Output = f64;
866
867            #[inline(always)]
868            fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
869                debug_assert_eq!(self.a.len(), self.b.len());
870                let (a_head, a_tail) = S::as_simd_f64s(self.a);
871                let (b_head, b_tail) = S::as_simd_f64s(self.b);
872                debug_assert_eq!(a_head.len(), b_head.len());
873                debug_assert_eq!(a_tail.len(), b_tail.len());
874
875                let mut acc0 = simd.splat_f64s(0.0);
876                let mut acc1 = simd.splat_f64s(0.0);
877                let mut acc2 = simd.splat_f64s(0.0);
878                let mut acc3 = simd.splat_f64s(0.0);
879
880                let mut i = 0usize;
881                while i + 4 <= a_head.len() {
882                    acc0 = simd.mul_add_f64s(a_head[i], b_head[i], acc0);
883                    acc1 = simd.mul_add_f64s(a_head[i + 1], b_head[i + 1], acc1);
884                    acc2 = simd.mul_add_f64s(a_head[i + 2], b_head[i + 2], acc2);
885                    acc3 = simd.mul_add_f64s(a_head[i + 3], b_head[i + 3], acc3);
886                    i += 4;
887                }
888                for j in i..a_head.len() {
889                    acc0 = simd.mul_add_f64s(a_head[j], b_head[j], acc0);
890                }
891
892                let acc = simd.add_f64s(simd.add_f64s(acc0, acc1), simd.add_f64s(acc2, acc3));
893                let mut sum = simd.reduce_sum_f64s(acc);
894                for (&x, &y) in a_tail.iter().zip(b_tail.iter()) {
895                    sum += x * y;
896                }
897                sum
898            }
899        }
900
901        Some(pulp::Arch::new().dispatch(Dot { a, b }))
902    }
903
904    impl MaybeSimdBlockFold for f64 {
905        fn try_simd_add_block(acc: &mut [f64], values: &[f64]) -> bool {
906            if acc.len() < BLOCK_FOLD_MIN_LEN {
907                return false;
908            }
909            struct AddBlock<'a, 'b>(&'a mut [f64], &'b [f64]);
910            impl pulp::WithSimd for AddBlock<'_, '_> {
911                type Output = ();
912                #[inline(always)]
913                fn with_simd<S: Simd>(self, simd: S) {
914                    let (ah, at) = S::as_mut_simd_f64s(self.0);
915                    let (vh, vt) = S::as_simd_f64s(self.1);
916                    // Equal block lengths are the trait contract, so the vector
917                    // prefix and the scalar tail cover the same indices.
918                    for (a, &v) in ah.iter_mut().zip(vh) {
919                        *a = simd.add_f64s(*a, v);
920                    }
921                    for (a, &v) in at.iter_mut().zip(vt) {
922                        *a += v;
923                    }
924                }
925            }
926            pulp::Arch::new().dispatch(AddBlock(acc, values));
927            true
928        }
929
930        fn try_simd_mul_block(acc: &mut [f64], values: &[f64]) -> bool {
931            if acc.len() < BLOCK_FOLD_MIN_LEN {
932                return false;
933            }
934            struct MulBlock<'a, 'b>(&'a mut [f64], &'b [f64]);
935            impl pulp::WithSimd for MulBlock<'_, '_> {
936                type Output = ();
937                #[inline(always)]
938                fn with_simd<S: Simd>(self, simd: S) {
939                    let (ah, at) = S::as_mut_simd_f64s(self.0);
940                    let (vh, vt) = S::as_simd_f64s(self.1);
941                    for (a, &v) in ah.iter_mut().zip(vh) {
942                        *a = simd.mul_f64s(*a, v);
943                    }
944                    for (a, &v) in at.iter_mut().zip(vt) {
945                        *a *= v;
946                    }
947                }
948            }
949            pulp::Arch::new().dispatch(MulBlock(acc, values));
950            true
951        }
952
953        fn try_simd_sum_squares_block(acc: &mut [f64], values: &[f64]) -> bool {
954            if acc.len() < BLOCK_FOLD_MIN_LEN {
955                return false;
956            }
957            struct SumSquaresBlock<'a, 'b>(&'a mut [f64], &'b [f64]);
958            impl pulp::WithSimd for SumSquaresBlock<'_, '_> {
959                type Output = ();
960                #[inline(always)]
961                fn with_simd<S: Simd>(self, simd: S) {
962                    let (ah, at) = S::as_mut_simd_f64s(self.0);
963                    let (vh, vt) = S::as_simd_f64s(self.1);
964                    for (a, &v) in ah.iter_mut().zip(vh) {
965                        let sq = simd.mul_f64s(v, v);
966                        *a = simd.add_f64s(*a, sq);
967                    }
968                    for (a, &v) in at.iter_mut().zip(vt) {
969                        *a += v * v;
970                    }
971                }
972            }
973            pulp::Arch::new().dispatch(SumSquaresBlock(acc, values));
974            true
975        }
976    }
977}
978
979#[cfg(test)]
980#[path = "simd/tests/tests.rs"]
981mod tests;