Skip to main content

wickra_simd/
lib.rs

1//! Runtime SIMD dispatch for the Wickra technical indicators core.
2//!
3//! Release binaries are built for the baseline of their target — on `x86_64`
4//! that is SSE2, with no AVX and no fused multiply-add instruction — so they
5//! run on every CPU of the architecture. A batch kernel that would profit from
6//! AVX2 lanes or a hardware FMA therefore cannot simply be compiled for them.
7//! [`dispatch`] closes that gap: it checks the CPU once (the standard library
8//! caches the answer) and runs a [`Kernel`] inside a function compiled with
9//! AVX2 and FMA enabled when both are present, and in the baseline build
10//! otherwise. A kernel that opts in ([`Kernel::WIDE`]) runs with AVX-512F
11//! enabled as well where the CPU has it, and does its element-wise work in
12//! eight-lane [`Simd::W`] values: one 512-bit register there, two four-lane
13//! vectors everywhere else.
14//!
15//! # Same result on every path
16//!
17//! Dispatch changes how a kernel is compiled, never what it computes. Rust
18//! evaluates floating-point code exactly as written — it never reassociates,
19//! and never contracts `a * b + c` into a fused multiply-add — so the AVX2
20//! build and the baseline build of one kernel perform the same IEEE-754
21//! operations in the same order. `f64::mul_add` is correctly rounded whether it
22//! becomes the FMA instruction or the C library's `fma`, and an eight-lane
23//! operation is the same IEEE-754 operation on each lane whether the lanes sit
24//! in one register or two. Every path therefore returns bit-identical values;
25//! the crate's tests check exactly that, at every level the machine has.
26//!
27//! The AVX-512 level needs Rust 1.89 or later, where its target feature is
28//! stable; built by an older compiler the crate has no such level, and a kernel
29//! that opts in runs at the AVX2 one -- the same bits, at the AVX2 speed.
30//!
31//! On `aarch64`, NEON and FMA are part of the baseline, so kernels already get
32//! them and [`dispatch`] runs them directly.
33//!
34//! # Writing a kernel
35//!
36//! Implement [`Kernel`] and mark [`Kernel::run`] `#[inline(always)]`, as well
37//! as any helper it calls: code is only compiled with the enabled features if
38//! it is inlined into the dispatching function.
39
40use std::sync::atomic::{AtomicU8, Ordering};
41
42mod avx2;
43// Compiled only when `build.rs` found Rust 1.89 or later, where the AVX-512
44// intrinsics are stable; the workspace MSRV builds the crate without it.
45#[clippy::msrv = "1.89"]
46mod avx512;
47mod portable;
48
49#[cfg(target_arch = "x86_64")]
50pub use avx2::Avx2;
51#[cfg(all(target_arch = "x86_64", wickra_avx512))]
52pub use avx512::Avx512;
53pub use portable::Portable;
54
55/// Four `f64` lanes and the operations the Wickra kernels are built from.
56///
57/// An implementation is a *token*: a zero-sized value whose existence proves
58/// the instruction set it stands for is available, so its methods are safe to
59/// call. [`Portable`] is available everywhere; [`Avx2`] is only ever created by
60/// [`dispatch`] after the CPU was checked.
61///
62/// Every operation is exact per lane in the IEEE-754 sense — add, subtract,
63/// multiply, divide, square root and fused multiply-add are correctly rounded,
64/// `max`/`min` follow the x86 `MAXPD`/`MINPD` rule (`a > b ? a : b`, so the
65/// second operand wins on a tie or a `NaN`) and `abs` clears the sign bit — so
66/// one kernel written against this trait returns the same bits under every
67/// implementation.
68pub trait Simd: Copy {
69    /// Four `f64` lanes.
70    type V: Copy;
71
72    /// All lanes `x`.
73    fn splat(self, x: f64) -> Self::V;
74    /// Lanes from an array.
75    fn load(self, a: &[f64; 4]) -> Self::V;
76    /// Lanes into an array.
77    fn store(self, v: Self::V, a: &mut [f64; 4]);
78    /// Lanes as an array.
79    fn to_array(self, v: Self::V) -> [f64; 4];
80    /// Lane-wise `a + b`.
81    fn add(self, a: Self::V, b: Self::V) -> Self::V;
82    /// Lane-wise `a - b`.
83    fn sub(self, a: Self::V, b: Self::V) -> Self::V;
84    /// Lane-wise `a * b`.
85    fn mul(self, a: Self::V, b: Self::V) -> Self::V;
86    /// Lane-wise `a / b`.
87    fn div(self, a: Self::V, b: Self::V) -> Self::V;
88    /// Lane-wise square root.
89    fn sqrt(self, a: Self::V) -> Self::V;
90    /// Lane-wise `a * b + c` with a single rounding.
91    fn mul_add(self, a: Self::V, b: Self::V, c: Self::V) -> Self::V;
92    /// Lane-wise `if a > b { a } else { b }`.
93    fn max(self, a: Self::V, b: Self::V) -> Self::V;
94    /// Lane-wise `if a < b { a } else { b }`.
95    fn min(self, a: Self::V, b: Self::V) -> Self::V;
96    /// Lane-wise absolute value (sign bit cleared).
97    fn abs(self, a: Self::V) -> Self::V;
98    /// Lane-wise `if test > 0 { yes } else { no }` (`NaN` takes `no`).
99    fn select_positive(self, test: Self::V, yes: Self::V, no: Self::V) -> Self::V;
100    /// `[0, v0, v1, v2]`: lanes moved up by one, `+0.0` shifted in.
101    fn shift1(self, v: Self::V) -> Self::V;
102    /// `[0, 0, v0, v1]`: lanes moved up by two, `+0.0` shifted in.
103    fn shift2(self, v: Self::V) -> Self::V;
104    /// All lanes the last lane `v3`.
105    fn broadcast_last(self, v: Self::V) -> Self::V;
106    /// The last lane `v3` as a scalar.
107    fn last_lane(self, v: Self::V) -> f64;
108    /// The 4×4 transpose of the rows `a`, `b`, `c`, `d`: result `j` is
109    /// `[a_j, b_j, c_j, d_j]`. Pure data movement, so no value changes.
110    fn transpose4(self, a: Self::V, b: Self::V, c: Self::V, d: Self::V) -> [Self::V; 4];
111
112    /// Eight `f64` lanes for element-wise work: one 512-bit register where the
113    /// CPU has AVX-512 and the kernel opts in ([`Kernel::WIDE`]), two
114    /// four-lane vectors otherwise. Every operation on them is exact per lane,
115    /// so the width never changes a result.
116    type W: Copy;
117
118    /// Eight lanes from an array.
119    fn load8(self, a: &[f64; 8]) -> Self::W;
120    /// Eight lanes into an array.
121    fn store8(self, v: Self::W, a: &mut [f64; 8]);
122    /// All eight lanes `x`.
123    fn splat8(self, x: f64) -> Self::W;
124    /// Lane-wise `a + b` over eight lanes.
125    fn add8(self, a: Self::W, b: Self::W) -> Self::W;
126    /// Lane-wise `a - b` over eight lanes.
127    fn sub8(self, a: Self::W, b: Self::W) -> Self::W;
128    /// Lane-wise `a * b` over eight lanes.
129    fn mul8(self, a: Self::W, b: Self::W) -> Self::W;
130    /// Lane-wise `a / b` over eight lanes.
131    fn div8(self, a: Self::W, b: Self::W) -> Self::W;
132    /// Lane-wise square root over eight lanes.
133    fn sqrt8(self, a: Self::W) -> Self::W;
134    /// Lane-wise `if a > b { a } else { b }` over eight lanes.
135    fn max8(self, a: Self::W, b: Self::W) -> Self::W;
136    /// Lane-wise `if test > 0 { yes } else { no }` over eight lanes (`NaN`
137    /// takes `no`).
138    fn select_positive8(self, test: Self::W, yes: Self::W, no: Self::W) -> Self::W;
139}
140
141/// A unit of work [`dispatch`] can run with the best instruction set available.
142///
143/// `run` receives a [`Simd`] token: vector kernels compute through it, scalar
144/// kernels ignore it and profit from the enabled instruction set through their
145/// plain arithmetic (a `mul_add` becomes the FMA instruction). Implementors mark
146/// [`run`](Kernel::run) `#[inline(always)]` so the body is compiled into the
147/// feature-enabled function rather than called out of it.
148pub trait Kernel {
149    /// What the kernel returns.
150    type Output;
151
152    /// Whether the kernel runs at the AVX-512 level where the CPU has it. A
153    /// kernel opts in when it does its element-wise work in eight lanes
154    /// ([`Simd::W`]) or its plain loops gain from the compiler widening them;
155    /// any other kernel runs at the AVX2 level on such a CPU, as code the
156    /// compiler vectorizes for AVX-512 is not always faster.
157    const WIDE: bool = false;
158
159    /// Execute the kernel with the lanes `simd` provides.
160    fn run<S: Simd>(self, simd: S) -> Self::Output;
161}
162
163/// The instruction set [`dispatch`] selects on this machine.
164#[derive(Debug, Clone, Copy, PartialEq, Eq)]
165pub enum Level {
166    /// Baseline instructions of the build target.
167    Baseline,
168    /// `x86_64` with AVX2 and FMA.
169    Avx2Fma,
170    /// `x86_64` with AVX2, FMA and AVX-512F: kernels that opt in
171    /// ([`Kernel::WIDE`]) get their eight-lane work in one register. Detected
172    /// only in a build by a Rust that supports AVX-512 (1.89 and later).
173    Avx512,
174    /// `aarch64`, whose baseline includes NEON and FMA.
175    Neon,
176}
177
178impl Level {
179    /// Stable lowercase name, for diagnostics and benchmark labels.
180    pub const fn name(self) -> &'static str {
181        match self {
182            Self::Baseline => "baseline",
183            Self::Avx2Fma => "avx2+fma",
184            Self::Avx512 => "avx512",
185            Self::Neon => "neon",
186        }
187    }
188}
189
190/// Cached detection result: 0 = not yet detected, 1 = baseline, 2 = AVX2+FMA,
191/// 3 = AVX-512.
192static DETECTED: AtomicU8 = AtomicU8::new(0);
193
194/// The instruction set [`dispatch`] runs kernels with on this machine.
195pub fn level() -> Level {
196    if cfg!(target_arch = "aarch64") {
197        return Level::Neon;
198    }
199    match DETECTED.load(Ordering::Relaxed) {
200        1 => Level::Baseline,
201        2 => Level::Avx2Fma,
202        3 => Level::Avx512,
203        _ => {
204            let level = detect();
205            let code = match level {
206                Level::Avx512 => 3,
207                Level::Avx2Fma => 2,
208                Level::Baseline | Level::Neon => 1,
209            };
210            DETECTED.store(code, Ordering::Relaxed);
211            level
212        }
213    }
214}
215
216#[cfg(target_arch = "x86_64")]
217fn detect() -> Level {
218    if std::arch::is_x86_feature_detected!("avx2") && std::arch::is_x86_feature_detected!("fma") {
219        if avx512() {
220            Level::Avx512
221        } else {
222            Level::Avx2Fma
223        }
224    } else {
225        Level::Baseline
226    }
227}
228
229/// Whether the CPU has AVX-512F, in a build that can use it.
230#[cfg(all(target_arch = "x86_64", wickra_avx512))]
231fn avx512() -> bool {
232    std::arch::is_x86_feature_detected!("avx512f")
233}
234
235#[cfg(all(target_arch = "x86_64", not(wickra_avx512)))]
236fn avx512() -> bool {
237    false
238}
239
240#[cfg(not(target_arch = "x86_64"))]
241fn detect() -> Level {
242    Level::Baseline
243}
244
245/// Run `kernel` with the best instruction set this CPU offers.
246///
247/// Returns bit-for-bit what [`run_baseline`] returns for the same kernel.
248#[inline]
249pub fn dispatch<K: Kernel>(kernel: K) -> K::Output {
250    run_level(level(), kernel)
251}
252
253/// `kernel` at `level`, which the caller has checked this CPU supports.
254#[inline]
255fn run_level<K: Kernel>(level: Level, kernel: K) -> K::Output {
256    match level {
257        #[cfg(all(target_arch = "x86_64", wickra_avx512))]
258        Level::Avx512 if K::WIDE => avx512::run(kernel),
259        #[cfg(target_arch = "x86_64")]
260        Level::Avx2Fma | Level::Avx512 => avx2::run(kernel),
261        _ => kernel.run(Portable),
262    }
263}
264
265/// Run `kernel` at `level` if this CPU supports it, `None` otherwise -- for
266/// tests that prove a kernel returns the same bits at every level the machine
267/// has, not only the one [`dispatch`] picks.
268#[doc(hidden)]
269pub fn run_at<K: Kernel>(level: Level, kernel: K) -> Option<K::Output> {
270    let best = self::level();
271    let available = match level {
272        Level::Baseline => true,
273        Level::Avx2Fma => matches!(best, Level::Avx2Fma | Level::Avx512),
274        Level::Avx512 | Level::Neon => best == level,
275    };
276    available.then(|| run_level(level, kernel))
277}
278
279/// Run `kernel` in the baseline build with the [`Portable`] lanes, never
280/// dispatching. Exists so tests (in this crate and in its users) can prove a
281/// kernel returns the same bits on every path.
282#[inline(never)]
283pub fn run_baseline<K: Kernel>(kernel: K) -> K::Output {
284    kernel.run(Portable)
285}
286
287#[cfg(test)]
288// Kernels must inline into the dispatching function to be compiled with its
289// features; `#[inline(always)]` on `run` is the documented contract.
290#[allow(clippy::inline_always)]
291mod tests {
292    use super::*;
293
294    /// Three interleaved EMA-style recurrences with `mul_add` plus a plain
295    /// multiply-add the compiler must not fuse: the shape of the scalar kernels.
296    struct Recurrences<'a> {
297        xs: &'a [f64],
298    }
299
300    impl Kernel for Recurrences<'_> {
301        type Output = Vec<f64>;
302
303        #[inline(always)]
304        fn run<S: Simd>(self, _simd: S) -> Vec<f64> {
305            let (mut a, mut b, mut c) = (1.0_f64, 2.0_f64, 3.0_f64);
306            let mut out = Vec::with_capacity(self.xs.len() * 3);
307            for &x in self.xs {
308                a = 0.1_f64.mul_add(x, 0.9 * a);
309                b = 0.3_f64.mul_add(x, 0.7 * b);
310                c = c * 0.95 + x * 0.05;
311                out.extend_from_slice(&[a, b, c - a]);
312            }
313            out
314        }
315    }
316
317    /// Every lane operation over one batch of inputs, the results concatenated.
318    struct AllOps<'a> {
319        a: &'a [f64],
320        b: &'a [f64],
321    }
322
323    impl Kernel for AllOps<'_> {
324        type Output = Vec<f64>;
325        const WIDE: bool = true;
326
327        #[inline(always)]
328        fn run<S: Simd>(self, s: S) -> Vec<f64> {
329            let mut out = Vec::new();
330            for (ca, cb) in self.a.chunks_exact(4).zip(self.b.chunks_exact(4)) {
331                let x = s.load(ca.try_into().unwrap());
332                let y = s.load(cb.try_into().unwrap());
333                let z = s.splat(ca[0]);
334                let mut stored = [0.0; 4];
335                s.store(s.add(x, y), &mut stored);
336                out.extend_from_slice(&stored);
337                for v in [
338                    s.sub(x, y),
339                    s.mul(x, y),
340                    s.div(x, y),
341                    s.sqrt(s.abs(x)),
342                    s.mul_add(x, y, z),
343                    s.max(x, y),
344                    s.min(x, y),
345                    s.abs(x),
346                    s.select_positive(s.sub(x, y), x, y),
347                    s.shift1(x),
348                    s.shift2(x),
349                    s.broadcast_last(x),
350                    s.splat(s.last_lane(y)),
351                    z,
352                ] {
353                    out.extend_from_slice(&s.to_array(v));
354                }
355                for v in s.transpose4(x, y, z, s.sub(x, y)) {
356                    out.extend_from_slice(&s.to_array(v));
357                }
358            }
359            for (ca, cb) in self.a.chunks_exact(8).zip(self.b.chunks_exact(8)) {
360                let x = s.load8(ca.try_into().unwrap());
361                let y = s.load8(cb.try_into().unwrap());
362                let z = s.splat8(ca[0]);
363                for v in [
364                    s.add8(x, y),
365                    s.sub8(x, y),
366                    s.mul8(x, y),
367                    s.div8(x, y),
368                    s.sqrt8(s.max8(x, z)),
369                    s.max8(x, y),
370                    s.select_positive8(s.sub8(x, y), x, y),
371                    z,
372                ] {
373                    let mut stored = [0.0; 8];
374                    s.store8(v, &mut stored);
375                    out.extend_from_slice(&stored);
376                }
377            }
378            out
379        }
380    }
381
382    fn bits(v: &[f64]) -> Vec<u64> {
383        v.iter().map(|x| x.to_bits()).collect()
384    }
385
386    fn series() -> Vec<f64> {
387        (0..4096)
388            .map(|i| (f64::from(i) * 0.731).sin() * 1e3 + f64::from(i % 17))
389            .collect()
390    }
391
392    #[test]
393    fn dispatch_matches_the_baseline_bit_for_bit() {
394        let xs = series();
395        assert_eq!(
396            bits(&dispatch(Recurrences { xs: &xs })),
397            bits(&run_baseline(Recurrences { xs: &xs }))
398        );
399    }
400
401    /// Each lane operation must give the same bits on every implementation,
402    /// the awkward values included: signed zeros, infinities, `NaN` (where
403    /// `max`/`min` must take the second operand), subnormals and a
404    /// cancellation-heavy fused multiply-add.
405    #[test]
406    fn every_lane_operation_matches_the_portable_lanes() {
407        let edge = [
408            0.0,
409            -0.0,
410            f64::INFINITY,
411            f64::NEG_INFINITY,
412            f64::NAN,
413            f64::MIN_POSITIVE / 8.0,
414            1.0 + f64::EPSILON,
415            -1.0,
416        ];
417        let mut a: Vec<f64> = series();
418        let mut b: Vec<f64> = series().iter().rev().map(|x| x * 0.37 - 2.0).collect();
419        for (i, &x) in edge.iter().enumerate() {
420            for (j, &y) in edge.iter().enumerate() {
421                a.extend_from_slice(&[x, y, 1.0 + f64::EPSILON, x]);
422                b.extend_from_slice(&[
423                    y,
424                    x,
425                    1.0 - f64::EPSILON,
426                    f64::from(u8::try_from(i * 8 + j).unwrap()),
427                ]);
428            }
429        }
430        let fast = dispatch(AllOps { a: &a, b: &b });
431        let base = run_baseline(AllOps { a: &a, b: &b });
432        assert_eq!(fast.len(), base.len());
433        for (k, (x, y)) in fast.iter().zip(&base).enumerate() {
434            assert!(
435                x.to_bits() == y.to_bits() || (x.is_nan() && y.is_nan()),
436                "lane {k}: {x:e} vs {y:e}"
437            );
438        }
439    }
440
441    #[test]
442    fn portable_lanes_follow_the_documented_rules() {
443        let s = Portable;
444        let x = [1.0, f64::NAN, -0.0, 3.0];
445        let y = [2.0, 5.0, 0.0, f64::NAN];
446        assert_eq!(bits(&s.max(x, y)), bits(&[2.0, 5.0, 0.0, f64::NAN]));
447        assert_eq!(bits(&s.min(x, y)), bits(&[1.0, 5.0, 0.0, f64::NAN]));
448        assert_eq!(s.shift1([1.0, 2.0, 3.0, 4.0]), [0.0, 1.0, 2.0, 3.0]);
449        assert_eq!(s.shift2([1.0, 2.0, 3.0, 4.0]), [0.0, 0.0, 1.0, 2.0]);
450        assert_eq!(s.broadcast_last([1.0, 2.0, 3.0, 4.0]), [4.0; 4]);
451        assert_eq!(
452            bits(&s.abs([-0.0, -2.0, 2.0, -f64::INFINITY])),
453            bits(&[0.0, 2.0, 2.0, f64::INFINITY])
454        );
455        assert_eq!(
456            s.transpose4(
457                [1.0, 2.0, 3.0, 4.0],
458                [5.0, 6.0, 7.0, 8.0],
459                [9.0, 10.0, 11.0, 12.0],
460                [13.0, 14.0, 15.0, 16.0]
461            ),
462            [
463                [1.0, 5.0, 9.0, 13.0],
464                [2.0, 6.0, 10.0, 14.0],
465                [3.0, 7.0, 11.0, 15.0],
466                [4.0, 8.0, 12.0, 16.0]
467            ]
468        );
469    }
470
471    #[test]
472    fn level_is_stable_and_named() {
473        let first = level();
474        assert_eq!(level(), first);
475        assert!(["baseline", "avx2+fma", "avx512", "neon"].contains(&first.name()));
476        assert_eq!(Level::Baseline.name(), "baseline");
477        assert_eq!(Level::Avx2Fma.name(), "avx2+fma");
478        assert_eq!(Level::Avx512.name(), "avx512");
479        assert_eq!(Level::Neon.name(), "neon");
480    }
481
482    #[cfg(target_arch = "x86_64")]
483    #[test]
484    fn detection_agrees_with_the_standard_library() {
485        let both = std::arch::is_x86_feature_detected!("avx2")
486            && std::arch::is_x86_feature_detected!("fma");
487        assert_eq!(matches!(level(), Level::Avx2Fma | Level::Avx512), both);
488        let wide = cfg!(wickra_avx512) && both && avx512();
489        assert_eq!(level() == Level::Avx512, wide);
490    }
491
492    /// Every level the machine has, and only those, runs a kernel -- with the
493    /// bits of the baseline build.
494    #[test]
495    fn every_available_level_matches_the_baseline() {
496        let a: Vec<f64> = series();
497        let b: Vec<f64> = series().iter().map(|x| x * 0.5 + 1.0).collect();
498        let base = run_baseline(AllOps { a: &a, b: &b });
499        let best = level();
500        for at in [Level::Baseline, Level::Avx2Fma, Level::Avx512, Level::Neon] {
501            let ran = run_at(at, AllOps { a: &a, b: &b });
502            let expected = match at {
503                Level::Baseline => true,
504                Level::Avx2Fma => matches!(best, Level::Avx2Fma | Level::Avx512),
505                Level::Avx512 | Level::Neon => best == at,
506            };
507            assert_eq!(ran.is_some(), expected, "{}", at.name());
508            if let Some(out) = ran {
509                assert!(out
510                    .iter()
511                    .zip(&base)
512                    .all(|(x, y)| x.to_bits() == y.to_bits() || (x.is_nan() && y.is_nan())));
513            }
514        }
515    }
516}