hermes_simd/dispatch/
mod.rs

1//! Runtime-dispatched SIMD operations.
2//!
3//! # Monomorphization chain
4//!
5//! `sum::<f32>(data)` -> `f32::sum(data)` -> `sum::dispatch_sum::<f32>(data)` -> avx2 kernel.
6
7mod abs_reduce;
8pub mod argmax;
9pub mod argmin;
10mod axpy;
11pub mod binary;
12pub mod complex;
13pub mod dot;
14pub mod gemm;
15pub mod gemv;
16pub mod gemv_strided;
17pub mod gemv_transpose;
18pub mod gemv_transpose_strided;
19pub mod masked;
20pub mod max;
21pub mod min;
22pub mod modular;
23mod popcount;
24pub mod scale;
25pub mod sparse;
26pub mod sum;
27
28pub use popcount::{
29    dispatch_reduce_popcount, dispatch_reduce_popcount_and, dispatch_reduce_popcount_or,
30    dispatch_reduce_popcount_xor,
31};
32
33use hermes_simd_core::scalar::Scalar as ScalarTrait;
34use hermes_simd_core::sparse::{
35    BlockedCooData, CsrData, DenseWithMaskData, SellPData, ValidatedData,
36};
37use hermes_simd_core::view::SimdError;
38use hermes_simd_core::{Add, Div, Mul, Sub};
39#[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
40use hermes_simd_intrinsics::Scalar as ScalarArch;
41#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
42#[allow(unused_imports)]
43use hermes_simd_intrinsics::{Avx2, Avx512, Neon, Scalar as ScalarArch};
44#[cfg(target_arch = "aarch64")]
45use hermes_simd_intrinsics::{Neon, Scalar as ScalarArch};
46
47mod private {
48    pub trait Sealed {}
49}
50
51impl private::Sealed for f32 {}
52impl private::Sealed for f64 {}
53impl private::Sealed for i8 {}
54impl private::Sealed for i16 {}
55impl private::Sealed for i32 {}
56
57impl private::Sealed for eunomia::F16 {}
58impl private::Sealed for eunomia::F32 {}
59impl private::Sealed for eunomia::F64 {}
60impl private::Sealed for eunomia::Bf16 {}
61impl private::Sealed for eunomia::Bf8 {}
62impl private::Sealed for eunomia::Bf4 {}
63impl private::Sealed for eunomia::F8 {}
64impl private::Sealed for eunomia::F4 {}
65impl private::Sealed for eunomia::I8 {}
66impl private::Sealed for eunomia::I16 {}
67impl private::Sealed for eunomia::I32 {}
68
69/// Sealed extension trait implementing dynamic runtime SIMD dispatch for any `T: Scalar`.
70pub trait SimdOps: ScalarTrait + private::Sealed {
71    /// Reduces the slice to its sum.
72    fn sum(data: &[Self]) -> Self;
73    /// Reduces the slice to `Σ |x|` (L1-norm accumulator); `T::ZERO` for empty.
74    fn abs_sum(data: &[Self]) -> Self;
75    /// Reduces the slice to `max |x|` (∞-norm accumulator); `T::ZERO` for empty.
76    fn abs_max(data: &[Self]) -> Self;
77    /// Reduces the slice to its minimum element.
78    ///
79    /// Returns `T::MAX_VALUE` for empty slices (the identity element for min).
80    fn min(data: &[Self]) -> Self;
81    /// Reduces the slice to its maximum element.
82    ///
83    /// Returns `T::MIN_VALUE` for empty slices (the identity element for max).
84    fn max(data: &[Self]) -> Self;
85    /// Multiplies every element by `scalar` in-place.
86    fn scale(data: &mut [Self], scalar: Self);
87    /// Returns `Some((index, value))` of the minimum element, or `None` for empty.
88    fn argmin(data: &[Self]) -> Option<(usize, Self)>;
89    /// Returns `Some((index, value))` of the maximum element, or `None` for empty.
90    fn argmax(data: &[Self]) -> Option<(usize, Self)>;
91    /// Computes the dot product of two slices.
92    fn dot(a: &[Self], b: &[Self]) -> Result<Self, SimdError>;
93    /// Fused row update `out[i] += alpha * x[i]` (AXPY) with no temporaries.
94    fn axpy(alpha: Self, x: &[Self], out: &mut [Self]) -> Result<(), SimdError>;
95    /// Fused multi-row update `out[row, i] += alphas[row] * x[i]`.
96    fn axpy_rows(
97        alphas: &[Self],
98        x: &[Self],
99        out: &mut [Self],
100        row_stride: usize,
101        rows: usize,
102        cols: usize,
103    ) -> Result<(), SimdError>;
104    /// Fused batched multi-row update:
105    /// `out[row, i] += sum_k alphas[k, row] * x_panel[k, i]`.
106    fn axpy_rows_batch(
107        alphas: &[Self],
108        x_panel: &[Self],
109        out: &mut [Self],
110        row_stride: usize,
111        rows: usize,
112        depth: usize,
113        cols: usize,
114    ) -> Result<(), SimdError>;
115    /// Computes the elementwise product and writes to `out`.
116    fn elementwise_mul(a: &[Self], b: &[Self], out: &mut [Self]) -> Result<(), SimdError>;
117    /// Computes the elementwise sum `a[i] + b[i]` and writes to `out`.
118    fn elementwise_add(a: &[Self], b: &[Self], out: &mut [Self]) -> Result<(), SimdError>;
119    /// Computes the elementwise difference `a[i] - b[i]` and writes to `out`.
120    fn elementwise_sub(a: &[Self], b: &[Self], out: &mut [Self]) -> Result<(), SimdError>;
121    /// Computes the elementwise quotient `a[i] / b[i]` and writes to `out`.
122    fn elementwise_div(a: &[Self], b: &[Self], out: &mut [Self]) -> Result<(), SimdError>;
123    /// Computes the sum of elements matching a boolean mask.
124    fn masked_sum(data: &[Self], mask: &[bool]) -> Self;
125    /// Computes the dot product of elements matching a boolean mask.
126    fn masked_dot(a: &[Self], b: &[Self], mask: &[bool]) -> Result<Self, SimdError>;
127    /// Computes the elementwise sum of elements matching a boolean mask.
128    fn masked_add(a: &[Self], b: &[Self], mask: &[bool], out: &mut [Self])
129        -> Result<(), SimdError>;
130    /// Computes sparse SpMV using CSR.
131    fn spmv_csr(data: ValidatedData<CsrData<'_, Self>>, x: &[Self], y: &mut [Self]);
132    /// Computes sparse SpMV using const-generic Blocked-COO tiles.
133    fn spmv_bcoo<const BM: usize, const BN: usize>(
134        data: ValidatedData<BlockedCooData<'_, Self, BM, BN>>,
135        x: &[Self],
136        y: &mut [Self],
137    );
138    /// Computes sparse SpMV using Dense-with-Mask.
139    fn spmv_dense_masked(data: DenseWithMaskData<'_, Self>, x: &[Self], y: &mut [Self]);
140    /// Computes sparse SpMV using const-generic Sliced ELLPACK (SELL-p).
141    fn spmv_sellp<const C: usize>(
142        data: ValidatedData<SellPData<'_, Self, C>>,
143        x: &[Self],
144        y: &mut [Self],
145    );
146    /// Computes register-blocked tiled GEMM: `c += A * B`.
147    fn tiled_gemm(
148        a: &[Self],
149        b: &[Self],
150        c: &mut [Self],
151        m: usize,
152        n: usize,
153        k: usize,
154    ) -> Result<(), SimdError>;
155    /// Computes register-blocked GEMV: `y += A * x` (`A` row-major `nrows × ncols`).
156    fn gemv(
157        a: &[Self],
158        x: &[Self],
159        y: &mut [Self],
160        nrows: usize,
161        ncols: usize,
162    ) -> Result<(), SimdError>;
163    /// Computes register-blocked transposed GEMV: `y += Aᵀ * x`
164    /// (`A` row-major `nrows × ncols`, `x` length `nrows`, `y` length `ncols`).
165    fn gemv_transpose(
166        a: &[Self],
167        x: &[Self],
168        y: &mut [Self],
169        nrows: usize,
170        ncols: usize,
171    ) -> Result<(), SimdError>;
172    /// Computes register-blocked sub-matrix GEMV: `y += A * x` with row stride
173    /// `lda ≥ ncols` (`lda = ncols` is the packed [`Self::gemv`]).
174    fn gemv_strided(
175        a: &[Self],
176        x: &[Self],
177        y: &mut [Self],
178        nrows: usize,
179        ncols: usize,
180        lda: usize,
181    ) -> Result<(), SimdError>;
182    /// Computes register-blocked transposed sub-matrix GEMV: `y += Aᵀ * x` with
183    /// row stride `lda ≥ ncols` (`lda = ncols` is the packed [`Self::gemv_transpose`]).
184    fn gemv_transpose_strided(
185        a: &[Self],
186        x: &[Self],
187        y: &mut [Self],
188        nrows: usize,
189        ncols: usize,
190        lda: usize,
191    ) -> Result<(), SimdError>;
192    /// Multiplies interleaved complex lanes in-place: `a[k] *= b[k]`
193    /// (`a[k] *= conj(b[k])` when `CONJ_B`).
194    fn interleaved_complex_mul_assign<const CONJ_B: bool>(
195        a: &mut [Self],
196        b: &[Self],
197    ) -> Result<(), SimdError>
198    where
199        Self: core::ops::Neg<Output = Self>;
200    /// Computes the interleaved complex dot product `(re, im)` of `sum(a[k] * b[k])`
201    /// (`sum(a[k] * conj(b[k]))` when `CONJ_B`).
202    fn interleaved_complex_dot<const CONJ_B: bool>(
203        a: &[Self],
204        b: &[Self],
205    ) -> Result<(Self, Self), SimdError>
206    where
207        Self: core::ops::Neg<Output = Self>;
208    /// Computes the horizontal sum of population counts of all elements.
209    fn reduce_popcount(data: &[Self]) -> usize;
210    /// Computes the horizontal sum of population counts of `a[i] & b[i]`.
211    fn reduce_popcount_and(a: &[Self], b: &[Self]) -> Result<usize, SimdError>;
212    /// Computes the horizontal sum of population counts of `a[i] | b[i]`.
213    fn reduce_popcount_or(a: &[Self], b: &[Self]) -> Result<usize, SimdError>;
214    /// Computes the horizontal sum of population counts of `a[i] ^ b[i]` (Hamming distance).
215    fn reduce_popcount_xor(a: &[Self], b: &[Self]) -> Result<usize, SimdError>;
216}
217
218/// Method bodies shared verbatim by the three target-gated `SimdOps`
219/// blanket impls below, which differ only in the architecture-kernel
220/// bound each `where` clause requires. Defining them once keeps the
221/// dispatch facade DRY and behavior identical across targets.
222macro_rules! impl_simd_ops_methods {
223    () => {
224        #[inline(always)]
225        fn sum(data: &[Self]) -> Self {
226            sum::dispatch_sum::<Self>(data)
227        }
228        #[inline(always)]
229        fn abs_sum(data: &[Self]) -> Self {
230            abs_reduce::dispatch_abs_sum::<Self>(data)
231        }
232        #[inline(always)]
233        fn abs_max(data: &[Self]) -> Self {
234            abs_reduce::dispatch_abs_max::<Self>(data)
235        }
236        #[inline(always)]
237        fn min(data: &[Self]) -> Self {
238            min::dispatch_min::<Self>(data)
239        }
240        #[inline(always)]
241        fn max(data: &[Self]) -> Self {
242            max::dispatch_max::<Self>(data)
243        }
244        #[inline(always)]
245        fn scale(data: &mut [Self], scalar: Self) {
246            scale::dispatch_scale::<Self>(data, scalar)
247        }
248        #[inline(always)]
249        fn argmin(data: &[Self]) -> Option<(usize, Self)> {
250            argmin::dispatch_argmin::<Self>(data)
251        }
252        #[inline(always)]
253        fn argmax(data: &[Self]) -> Option<(usize, Self)> {
254            argmax::dispatch_argmax::<Self>(data)
255        }
256        #[inline(always)]
257        fn dot(a: &[Self], b: &[Self]) -> Result<Self, SimdError> {
258            dot::dispatch_dot::<Self>(a, b)
259        }
260        #[inline(always)]
261        fn axpy(alpha: Self, x: &[Self], out: &mut [Self]) -> Result<(), SimdError> {
262            axpy::dispatch_axpy::<Self>(alpha, x, out)
263        }
264        #[inline(always)]
265        fn axpy_rows(
266            alphas: &[Self],
267            x: &[Self],
268            out: &mut [Self],
269            row_stride: usize,
270            rows: usize,
271            cols: usize,
272        ) -> Result<(), SimdError> {
273            axpy::dispatch_axpy_rows::<Self>(alphas, x, out, row_stride, rows, cols)
274        }
275        #[inline(always)]
276        fn axpy_rows_batch(
277            alphas: &[Self],
278            x_panel: &[Self],
279            out: &mut [Self],
280            row_stride: usize,
281            rows: usize,
282            depth: usize,
283            cols: usize,
284        ) -> Result<(), SimdError> {
285            axpy::dispatch_axpy_rows_batch::<Self>(
286                alphas, x_panel, out, row_stride, rows, depth, cols,
287            )
288        }
289        #[inline(always)]
290        fn elementwise_mul(a: &[Self], b: &[Self], out: &mut [Self]) -> Result<(), SimdError> {
291            binary::dispatch_elementwise_binary::<Self, Mul>(a, b, out, Mul)
292        }
293        #[inline(always)]
294        fn elementwise_add(a: &[Self], b: &[Self], out: &mut [Self]) -> Result<(), SimdError> {
295            binary::dispatch_elementwise_binary::<Self, Add>(a, b, out, Add)
296        }
297        #[inline(always)]
298        fn elementwise_sub(a: &[Self], b: &[Self], out: &mut [Self]) -> Result<(), SimdError> {
299            binary::dispatch_elementwise_binary::<Self, Sub>(a, b, out, Sub)
300        }
301        #[inline(always)]
302        fn elementwise_div(a: &[Self], b: &[Self], out: &mut [Self]) -> Result<(), SimdError> {
303            binary::dispatch_elementwise_binary::<Self, Div>(a, b, out, Div)
304        }
305        #[inline(always)]
306        fn masked_sum(data: &[Self], mask: &[bool]) -> Self {
307            masked::dispatch_masked_sum::<Self>(data, mask)
308        }
309        #[inline(always)]
310        fn masked_dot(a: &[Self], b: &[Self], mask: &[bool]) -> Result<Self, SimdError> {
311            masked::dispatch_masked_dot::<Self>(a, b, mask)
312        }
313        #[inline(always)]
314        fn masked_add(
315            a: &[Self],
316            b: &[Self],
317            mask: &[bool],
318            out: &mut [Self],
319        ) -> Result<(), SimdError> {
320            masked::dispatch_masked_add::<Self>(a, b, mask, out)
321        }
322        #[inline(always)]
323        fn spmv_csr(data: ValidatedData<CsrData<'_, Self>>, x: &[Self], y: &mut [Self]) {
324            sparse::dispatch_spmv_csr::<Self>(data, x, y)
325        }
326        #[inline(always)]
327        fn spmv_bcoo<const BM: usize, const BN: usize>(
328            data: ValidatedData<BlockedCooData<'_, Self, BM, BN>>,
329            x: &[Self],
330            y: &mut [Self],
331        ) {
332            // Runtime-dispatched like the other sparse kernels (was hardcoded to
333            // ScalarArch, which left the SIMD BlockedCoo paths dead at runtime).
334            sparse::dispatch_spmv_bcoo::<Self, BM, BN>(data, x, y)
335        }
336        #[inline(always)]
337        fn spmv_dense_masked(data: DenseWithMaskData<'_, Self>, x: &[Self], y: &mut [Self]) {
338            sparse::dispatch_spmv_dense_masked::<Self>(data, x, y)
339        }
340        #[inline(always)]
341        fn spmv_sellp<const C: usize>(
342            data: ValidatedData<SellPData<'_, Self, C>>,
343            x: &[Self],
344            y: &mut [Self],
345        ) {
346            sparse::dispatch_spmv_sellp::<Self, C>(data, x, y)
347        }
348        #[inline(always)]
349        fn tiled_gemm(
350            a: &[Self],
351            b: &[Self],
352            c: &mut [Self],
353            m: usize,
354            n: usize,
355            k: usize,
356        ) -> Result<(), SimdError> {
357            gemm::dispatch_tiled_gemm::<Self>(a, b, c, m, n, k)
358        }
359        #[inline(always)]
360        fn gemv(
361            a: &[Self],
362            x: &[Self],
363            y: &mut [Self],
364            nrows: usize,
365            ncols: usize,
366        ) -> Result<(), SimdError> {
367            gemv::dispatch_gemv::<Self>(a, x, y, nrows, ncols)
368        }
369        #[inline(always)]
370        fn gemv_transpose(
371            a: &[Self],
372            x: &[Self],
373            y: &mut [Self],
374            nrows: usize,
375            ncols: usize,
376        ) -> Result<(), SimdError> {
377            gemv_transpose::dispatch_gemv_transpose::<Self>(a, x, y, nrows, ncols)
378        }
379        #[inline(always)]
380        fn gemv_strided(
381            a: &[Self],
382            x: &[Self],
383            y: &mut [Self],
384            nrows: usize,
385            ncols: usize,
386            lda: usize,
387        ) -> Result<(), SimdError> {
388            gemv_strided::dispatch_gemv_strided::<Self>(a, x, y, nrows, ncols, lda)
389        }
390        #[inline(always)]
391        fn gemv_transpose_strided(
392            a: &[Self],
393            x: &[Self],
394            y: &mut [Self],
395            nrows: usize,
396            ncols: usize,
397            lda: usize,
398        ) -> Result<(), SimdError> {
399            gemv_transpose_strided::dispatch_gemv_transpose_strided::<Self>(
400                a, x, y, nrows, ncols, lda,
401            )
402        }
403        #[inline(always)]
404        fn interleaved_complex_mul_assign<const CONJ_B: bool>(
405            a: &mut [Self],
406            b: &[Self],
407        ) -> Result<(), SimdError>
408        where
409            Self: core::ops::Neg<Output = Self>,
410        {
411            complex::dispatch_interleaved_complex_mul_assign::<Self, CONJ_B>(a, b)
412        }
413        #[inline(always)]
414        fn interleaved_complex_dot<const CONJ_B: bool>(
415            a: &[Self],
416            b: &[Self],
417        ) -> Result<(Self, Self), SimdError>
418        where
419            Self: core::ops::Neg<Output = Self>,
420        {
421            complex::dispatch_interleaved_complex_dot::<Self, CONJ_B>(a, b)
422        }
423        #[inline(always)]
424        fn reduce_popcount(data: &[Self]) -> usize {
425            dispatch_reduce_popcount::<Self>(data)
426        }
427        #[inline(always)]
428        fn reduce_popcount_and(a: &[Self], b: &[Self]) -> Result<usize, SimdError> {
429            dispatch_reduce_popcount_and::<Self>(a, b)
430        }
431        #[inline(always)]
432        fn reduce_popcount_or(a: &[Self], b: &[Self]) -> Result<usize, SimdError> {
433            dispatch_reduce_popcount_or::<Self>(a, b)
434        }
435        #[inline(always)]
436        fn reduce_popcount_xor(a: &[Self], b: &[Self]) -> Result<usize, SimdError> {
437            dispatch_reduce_popcount_xor::<Self>(a, b)
438        }
439    };
440}
441
442/// x86/x86_64 specialized generic implementation of SimdOps.
443#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
444impl<T> SimdOps for T
445where
446    T: ScalarTrait + private::Sealed,
447    ScalarArch: hermes_simd_core::kernel::SimdKernel<T>,
448    Avx2: hermes_simd_core::kernel::SimdKernel<T>,
449    Avx512: hermes_simd_core::kernel::SimdKernel<T>,
450{
451    impl_simd_ops_methods!();
452}
453
454/// AArch64 specialized generic implementation of SimdOps.
455#[cfg(target_arch = "aarch64")]
456impl<T> SimdOps for T
457where
458    T: ScalarTrait + private::Sealed,
459    ScalarArch: hermes_simd_core::kernel::SimdKernel<T>,
460    Neon: hermes_simd_core::kernel::SimdKernel<T>,
461{
462    impl_simd_ops_methods!();
463}
464
465/// Fallback generic implementation of SimdOps.
466#[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
467impl<T> SimdOps for T
468where
469    T: ScalarTrait + private::Sealed,
470    ScalarArch: hermes_simd_core::kernel::SimdKernel<T>,
471{
472    impl_simd_ops_methods!();
473}
474
475/// Computes the sum of elements in the slice using runtime-dispatched SIMD.
476#[inline(always)]
477pub fn sum<T: SimdOps>(data: &[T]) -> T {
478    T::sum(data)
479}
480
481/// Computes the minimum element of the slice using runtime-dispatched SIMD.
482///
483/// Returns `T::MAX_VALUE` for empty slices.
484#[inline(always)]
485pub fn min<T: SimdOps>(data: &[T]) -> T {
486    T::min(data)
487}
488
489/// Computes the maximum element of the slice using runtime-dispatched SIMD.
490///
491/// Returns `T::MIN_VALUE` for empty slices.
492#[inline(always)]
493pub fn max<T: SimdOps>(data: &[T]) -> T {
494    T::max(data)
495}
496
497/// Reduces the slice to `Σ |x|` (L1-norm accumulator); `T::ZERO` for empty.
498#[inline(always)]
499pub fn abs_sum<T: SimdOps>(data: &[T]) -> T {
500    T::abs_sum(data)
501}
502
503/// Reduces the slice to `max |x|` (∞-norm accumulator); `T::ZERO` for empty.
504#[inline(always)]
505pub fn abs_max<T: SimdOps>(data: &[T]) -> T {
506    T::abs_max(data)
507}
508
509/// Multiplies every element of `data` by `scalar` in-place.
510#[inline(always)]
511pub fn scale<T: SimdOps>(data: &mut [T], scalar: T) {
512    T::scale(data, scalar)
513}
514
515/// Returns the first minimum, or `None` for empty or NaN-containing data.
516#[inline(always)]
517pub fn argmin<T: SimdOps>(data: &[T]) -> Option<(usize, T)> {
518    T::argmin(data)
519}
520
521/// Returns the first maximum, or `None` for empty or NaN-containing data.
522#[inline(always)]
523pub fn argmax<T: SimdOps>(data: &[T]) -> Option<(usize, T)> {
524    T::argmax(data)
525}
526
527/// Computes the dot product of two slices using runtime-dispatched SIMD.
528#[inline(always)]
529pub fn dot<T: SimdOps>(a: &[T], b: &[T]) -> Result<T, SimdError> {
530    T::dot(a, b)
531}
532
533/// Fused row update `out[i] += alpha * x[i]` (AXPY) via runtime-dispatched
534/// SIMD with no temporary allocation. Errors on length mismatch.
535#[inline(always)]
536pub fn axpy<T: SimdOps>(alpha: T, x: &[T], out: &mut [T]) -> Result<(), SimdError> {
537    T::axpy(alpha, x, out)
538}
539
540/// Fused multi-row update `out[row, i] += alphas[row] * x[i]` via one
541/// runtime-dispatched SIMD kernel. `out` is a row-major strided window.
542#[inline(always)]
543pub fn axpy_rows<T: SimdOps>(
544    alphas: &[T],
545    x: &[T],
546    out: &mut [T],
547    row_stride: usize,
548    rows: usize,
549    cols: usize,
550) -> Result<(), SimdError> {
551    T::axpy_rows(alphas, x, out, row_stride, rows, cols)
552}
553
554/// Fused batched multi-row update:
555/// `out[row, i] += sum_k alphas[k, row] * x_panel[k, i]` via one
556/// runtime-dispatched SIMD kernel. `alphas` is depth-major with `rows`
557/// elements per depth, `x_panel` is depth-major with `cols` elements per
558/// depth, and `out` is a row-major strided window.
559#[inline(always)]
560pub fn axpy_rows_batch<T: SimdOps>(
561    alphas: &[T],
562    x_panel: &[T],
563    out: &mut [T],
564    row_stride: usize,
565    rows: usize,
566    depth: usize,
567    cols: usize,
568) -> Result<(), SimdError> {
569    T::axpy_rows_batch(alphas, x_panel, out, row_stride, rows, depth, cols)
570}
571
572/// Computes the elementwise multiplication of two slices and writes to `out`.
573#[inline(always)]
574pub fn elementwise_mul<T: SimdOps>(a: &[T], b: &[T], out: &mut [T]) -> Result<(), SimdError> {
575    T::elementwise_mul(a, b, out)
576}
577
578/// Computes the elementwise sum of two slices and writes to `out`.
579#[inline(always)]
580pub fn elementwise_add<T: SimdOps>(a: &[T], b: &[T], out: &mut [T]) -> Result<(), SimdError> {
581    T::elementwise_add(a, b, out)
582}
583
584/// Computes the elementwise difference of two slices and writes to `out`.
585#[inline(always)]
586pub fn elementwise_sub<T: SimdOps>(a: &[T], b: &[T], out: &mut [T]) -> Result<(), SimdError> {
587    T::elementwise_sub(a, b, out)
588}
589
590/// Computes the elementwise quotient of two slices and writes to `out`.
591#[inline(always)]
592pub fn elementwise_div<T: SimdOps>(a: &[T], b: &[T], out: &mut [T]) -> Result<(), SimdError> {
593    T::elementwise_div(a, b, out)
594}
595
596/// Executes one exact modular radix-2 NTT butterfly stage over `u64` residues.
597#[inline]
598pub fn ntt_butterfly_stage_u64(
599    data: &mut [u64],
600    stage_len: usize,
601    twiddles: &[u64],
602    modulus: u64,
603) -> Result<(), SimdError> {
604    modular::ntt_butterfly_stage_u64(data, stage_len, twiddles, modulus)
605}
606
607/// Computes the sum of elements matching a boolean mask.
608#[inline(always)]
609pub fn masked_sum<T: SimdOps>(data: &[T], mask: &[bool]) -> T {
610    T::masked_sum(data, mask)
611}
612
613/// Computes the dot product of elements matching a boolean mask.
614#[inline(always)]
615pub fn masked_dot<T: SimdOps>(a: &[T], b: &[T], mask: &[bool]) -> Result<T, SimdError> {
616    T::masked_dot(a, b, mask)
617}
618
619/// Computes the elementwise sum of elements matching a boolean mask.
620#[inline(always)]
621pub fn masked_add<T: SimdOps>(
622    a: &[T],
623    b: &[T],
624    mask: &[bool],
625    out: &mut [T],
626) -> Result<(), SimdError> {
627    T::masked_add(a, b, mask, out)
628}
629
630/// Computes sparse SpMV using CSR: `y += A · x`.
631///
632/// # Panics
633/// Panics if `x.len() < ncols` or `y.len() < nrows`. Structural CSR validation
634/// is performed by [`ValidatedData::new`] before this function can be called.
635#[inline(always)]
636pub fn spmv_csr<T: SimdOps>(data: ValidatedData<CsrData<'_, T>>, x: &[T], y: &mut [T]) {
637    T::spmv_csr(data, x, y)
638}
639
640/// Computes sparse SpMV using const-generic Blocked-COO tiles.
641///
642/// # Panics
643/// Panics if `x.len() < ncols` or `y.len() < nrows`. Structural Blocked-COO
644/// validation is performed by [`ValidatedData::new`] before this function can be
645/// called.
646#[inline(always)]
647pub fn spmv_bcoo<T: SimdOps, const BM: usize, const BN: usize>(
648    data: ValidatedData<BlockedCooData<'_, T, BM, BN>>,
649    x: &[T],
650    y: &mut [T],
651) {
652    T::spmv_bcoo::<BM, BN>(data, x, y)
653}
654
655/// Computes sparse SpMV using Dense-with-Mask.
656#[inline(always)]
657pub fn spmv_dense_masked<T: SimdOps>(data: DenseWithMaskData<'_, T>, x: &[T], y: &mut [T]) {
658    T::spmv_dense_masked(data, x, y)
659}
660
661/// Computes sparse SpMV using const-generic Sliced ELLPACK (SELL-p).
662///
663/// # Panics
664/// Panics if `x.len() < ncols` or `y.len() < nrows`. Structural SELL-p
665/// validation is performed by [`ValidatedData::new`] before this function can be
666/// called.
667#[inline(always)]
668pub fn spmv_sellp<T: SimdOps, const C: usize>(
669    data: ValidatedData<SellPData<'_, T, C>>,
670    x: &[T],
671    y: &mut [T],
672) {
673    T::spmv_sellp::<C>(data, x, y)
674}
675
676/// Computes register-blocked tiled GEMM: `c += A * B`.
677#[inline(always)]
678pub fn tiled_gemm<T: SimdOps>(
679    a: &[T],
680    b: &[T],
681    c: &mut [T],
682    m: usize,
683    n: usize,
684    k: usize,
685) -> Result<(), SimdError> {
686    T::tiled_gemm(a, b, c, m, n, k)
687}
688
689/// Computes register-blocked GEMV `y += A · x` with runtime backend selection.
690///
691/// `a` is row-major `nrows × ncols`; the product **accumulates** into `y`
692/// (zero `y` first for `y = A·x`). See [`gemv()`] for the
693/// operand-reuse theorem.
694///
695/// # Errors
696/// [`SimdError::LengthMismatch`] if `a.len() < nrows·ncols`, `x.len() < ncols`,
697/// or `y.len() < nrows`.
698#[inline(always)]
699pub fn gemv<T: SimdOps>(
700    a: &[T],
701    x: &[T],
702    y: &mut [T],
703    nrows: usize,
704    ncols: usize,
705) -> Result<(), SimdError> {
706    T::gemv(a, x, y, nrows, ncols)
707}
708
709/// Computes register-blocked transposed GEMV `y += Aᵀ · x` with runtime backend
710/// selection — the complement of [`gemv()`].
711///
712/// `a` is row-major `nrows × ncols`, `x` length `nrows`, `y` length `ncols`; the
713/// product **accumulates** into `y` (zero `y` first for `y = Aᵀ·x`). See
714/// [`gemv_transpose()`] for the operand-reuse theorem.
715///
716/// # Errors
717/// [`SimdError::LengthMismatch`] if `a.len() < nrows·ncols`, `x.len() < nrows`,
718/// or `y.len() < ncols`.
719#[inline(always)]
720pub fn gemv_transpose<T: SimdOps>(
721    a: &[T],
722    x: &[T],
723    y: &mut [T],
724    nrows: usize,
725    ncols: usize,
726) -> Result<(), SimdError> {
727    T::gemv_transpose(a, x, y, nrows, ncols)
728}
729
730/// Computes register-blocked sub-matrix GEMV `y += A · x` with row stride `lda`,
731/// runtime backend selection. `A` is a row-major `nrows × ncols` block with
732/// leading dimension `lda ≥ ncols`; `lda = ncols` is the packed [`gemv()`].
733/// Accumulates into `y`.
734///
735/// # Errors
736/// [`SimdError::LengthMismatch`] if `lda < ncols`, `a.len() < (nrows−1)·lda +
737/// ncols`, `x.len() < ncols`, or `y.len() < nrows`.
738#[inline(always)]
739pub fn gemv_strided<T: SimdOps>(
740    a: &[T],
741    x: &[T],
742    y: &mut [T],
743    nrows: usize,
744    ncols: usize,
745    lda: usize,
746) -> Result<(), SimdError> {
747    T::gemv_strided(a, x, y, nrows, ncols, lda)
748}
749
750/// Computes register-blocked transposed sub-matrix GEMV `y += Aᵀ · x` with row
751/// stride `lda`, runtime backend selection. `lda = ncols` is the packed
752/// [`gemv_transpose()`]. Accumulates into `y`.
753///
754/// # Errors
755/// [`SimdError::LengthMismatch`] if `lda < ncols`, `a.len() < (nrows−1)·lda +
756/// ncols`, `x.len() < nrows`, or `y.len() < ncols`.
757#[inline(always)]
758pub fn gemv_transpose_strided<T: SimdOps>(
759    a: &[T],
760    x: &[T],
761    y: &mut [T],
762    nrows: usize,
763    ncols: usize,
764    lda: usize,
765) -> Result<(), SimdError> {
766    T::gemv_transpose_strided(a, x, y, nrows, ncols, lda)
767}
768
769/// Multiplies interleaved complex values in-place using a monomorphized SIMD architecture.
770///
771/// Inputs are primitive lane slices in `[re0, im0, re1, im1, ...]` order. `a`
772/// is updated with `a[i] * b[i]`; when `CONJ_B` is true, the operation is
773/// `a[i] * conj(b[i])`.
774#[inline]
775pub fn interleaved_complex_mul_assign<T, A, const CONJ_B: bool>(
776    a: &mut [T],
777    b: &[T],
778) -> Result<(), SimdError>
779where
780    T: ScalarTrait + core::ops::Neg<Output = T>,
781    A: hermes_simd_core::arch::SimdArch + hermes_simd_core::kernel::SimdKernel<T>,
782{
783    complex::interleaved_complex_mul_assign::<T, A, CONJ_B>(a, b)
784}
785
786/// Computes an interleaved complex dot product using a monomorphized SIMD architecture.
787///
788/// Inputs are primitive lane slices in `[re0, im0, re1, im1, ...]` order. The
789/// result is `(re, im)` for `sum(a[i] * b[i])`; when `CONJ_B` is true, the
790/// operation is `sum(a[i] * conj(b[i]))`.
791#[inline]
792pub fn interleaved_complex_dot<T, A, const CONJ_B: bool>(
793    a: &[T],
794    b: &[T],
795) -> Result<(T, T), SimdError>
796where
797    T: ScalarTrait + core::ops::Neg<Output = T>,
798    A: hermes_simd_core::arch::SimdArch + hermes_simd_core::kernel::SimdKernel<T>,
799{
800    complex::interleaved_complex_dot::<T, A, CONJ_B>(a, b)
801}
802
803/// Multiplies interleaved complex values in-place using Hermes runtime provider selection.
804#[inline]
805pub fn interleaved_complex_mul_assign_runtime<T, const CONJ_B: bool>(
806    a: &mut [T],
807    b: &[T],
808) -> Result<(), SimdError>
809where
810    T: SimdOps + core::ops::Neg<Output = T>,
811{
812    T::interleaved_complex_mul_assign::<CONJ_B>(a, b)
813}
814
815/// Computes an interleaved complex dot product using Hermes runtime provider selection.
816#[inline]
817pub fn interleaved_complex_dot_runtime<T, const CONJ_B: bool>(
818    a: &[T],
819    b: &[T],
820) -> Result<(T, T), SimdError>
821where
822    T: SimdOps + core::ops::Neg<Output = T>,
823{
824    T::interleaved_complex_dot::<CONJ_B>(a, b)
825}
826
827/// Computes the horizontal sum of population counts of all elements using runtime-dispatched SIMD.
828#[inline(always)]
829pub fn reduce_popcount<T: SimdOps>(data: &[T]) -> usize {
830    T::reduce_popcount(data)
831}
832
833/// Computes the horizontal sum of population counts of `a[i] & b[i]` using runtime-dispatched SIMD.
834#[inline(always)]
835pub fn reduce_popcount_and<T: SimdOps>(a: &[T], b: &[T]) -> Result<usize, SimdError> {
836    T::reduce_popcount_and(a, b)
837}
838
839/// Computes the horizontal sum of population counts of `a[i] | b[i]` using runtime-dispatched SIMD.
840#[inline(always)]
841pub fn reduce_popcount_or<T: SimdOps>(a: &[T], b: &[T]) -> Result<usize, SimdError> {
842    T::reduce_popcount_or(a, b)
843}
844
845/// Computes the horizontal sum of population counts of `a[i] ^ b[i]` (Hamming distance) using runtime-dispatched SIMD.
846#[inline(always)]
847pub fn reduce_popcount_xor<T: SimdOps>(a: &[T], b: &[T]) -> Result<usize, SimdError> {
848    T::reduce_popcount_xor(a, b)
849}
hermes_simd/dispatch/mod.rs

hermes_simd/dispatch/
mod.rs
