wickra_simd/lib.rs
1//! Runtime SIMD dispatch for the Wickra technical indicators core.
2//!
3//! Release binaries are built for the baseline of their target — on `x86_64`
4//! that is SSE2, with no AVX and no fused multiply-add instruction — so they
5//! run on every CPU of the architecture. A batch kernel that would profit from
6//! AVX2 lanes or a hardware FMA therefore cannot simply be compiled for them.
7//! [`dispatch`] closes that gap: it checks the CPU once (the standard library
8//! caches the answer) and runs a [`Kernel`] inside a function compiled with
9//! AVX2 and FMA enabled when both are present, and in the baseline build
10//! otherwise. A kernel that opts in ([`Kernel::WIDE`]) runs with AVX-512F
11//! enabled as well where the CPU has it, and does its element-wise work in
12//! eight-lane [`Simd::W`] values: one 512-bit register there, two four-lane
13//! vectors everywhere else.
14//!
15//! # Same result on every path
16//!
17//! Dispatch changes how a kernel is compiled, never what it computes. Rust
18//! evaluates floating-point code exactly as written — it never reassociates,
19//! and never contracts `a * b + c` into a fused multiply-add — so the AVX2
20//! build and the baseline build of one kernel perform the same IEEE-754
21//! operations in the same order. `f64::mul_add` is correctly rounded whether it
22//! becomes the FMA instruction or the C library's `fma`, and an eight-lane
23//! operation is the same IEEE-754 operation on each lane whether the lanes sit
24//! in one register or two. Every path therefore returns bit-identical values;
25//! the crate's tests check exactly that, at every level the machine has.
26//!
27//! The AVX-512 level needs Rust 1.89 or later, where its target feature is
28//! stable; built by an older compiler the crate has no such level, and a kernel
29//! that opts in runs at the AVX2 one -- the same bits, at the AVX2 speed.
30//!
31//! On `aarch64`, NEON and FMA are part of the baseline, so kernels already get
32//! them and [`dispatch`] runs them directly.
33//!
34//! # Writing a kernel
35//!
36//! Implement [`Kernel`] and mark [`Kernel::run`] `#[inline(always)]`, as well
37//! as any helper it calls: code is only compiled with the enabled features if
38//! it is inlined into the dispatching function.
39
40use std::sync::atomic::{AtomicU8, Ordering};
41
42mod avx2;
43// Compiled only when `build.rs` found Rust 1.89 or later, where the AVX-512
44// intrinsics are stable; the workspace MSRV builds the crate without it.
45#[clippy::msrv = "1.89"]
46mod avx512;
47mod portable;
48
49#[cfg(target_arch = "x86_64")]
50pub use avx2::Avx2;
51#[cfg(all(target_arch = "x86_64", wickra_avx512))]
52pub use avx512::Avx512;
53pub use portable::Portable;
54
55/// Four `f64` lanes and the operations the Wickra kernels are built from.
56///
57/// An implementation is a *token*: a zero-sized value whose existence proves
58/// the instruction set it stands for is available, so its methods are safe to
59/// call. [`Portable`] is available everywhere; [`Avx2`] is only ever created by
60/// [`dispatch`] after the CPU was checked.
61///
62/// Every operation is exact per lane in the IEEE-754 sense — add, subtract,
63/// multiply, divide, square root and fused multiply-add are correctly rounded,
64/// `max`/`min` follow the x86 `MAXPD`/`MINPD` rule (`a > b ? a : b`, so the
65/// second operand wins on a tie or a `NaN`) and `abs` clears the sign bit — so
66/// one kernel written against this trait returns the same bits under every
67/// implementation.
68pub trait Simd: Copy {
69 /// Four `f64` lanes.
70 type V: Copy;
71
72 /// All lanes `x`.
73 fn splat(self, x: f64) -> Self::V;
74 /// Lanes from an array.
75 fn load(self, a: &[f64; 4]) -> Self::V;
76 /// Lanes into an array.
77 fn store(self, v: Self::V, a: &mut [f64; 4]);
78 /// Lanes as an array.
79 fn to_array(self, v: Self::V) -> [f64; 4];
80 /// Lane-wise `a + b`.
81 fn add(self, a: Self::V, b: Self::V) -> Self::V;
82 /// Lane-wise `a - b`.
83 fn sub(self, a: Self::V, b: Self::V) -> Self::V;
84 /// Lane-wise `a * b`.
85 fn mul(self, a: Self::V, b: Self::V) -> Self::V;
86 /// Lane-wise `a / b`.
87 fn div(self, a: Self::V, b: Self::V) -> Self::V;
88 /// Lane-wise square root.
89 fn sqrt(self, a: Self::V) -> Self::V;
90 /// Lane-wise `a * b + c` with a single rounding.
91 fn mul_add(self, a: Self::V, b: Self::V, c: Self::V) -> Self::V;
92 /// Lane-wise `if a > b { a } else { b }`.
93 fn max(self, a: Self::V, b: Self::V) -> Self::V;
94 /// Lane-wise `if a < b { a } else { b }`.
95 fn min(self, a: Self::V, b: Self::V) -> Self::V;
96 /// Lane-wise absolute value (sign bit cleared).
97 fn abs(self, a: Self::V) -> Self::V;
98 /// Lane-wise `if test > 0 { yes } else { no }` (`NaN` takes `no`).
99 fn select_positive(self, test: Self::V, yes: Self::V, no: Self::V) -> Self::V;
100 /// `[0, v0, v1, v2]`: lanes moved up by one, `+0.0` shifted in.
101 fn shift1(self, v: Self::V) -> Self::V;
102 /// `[0, 0, v0, v1]`: lanes moved up by two, `+0.0` shifted in.
103 fn shift2(self, v: Self::V) -> Self::V;
104 /// All lanes the last lane `v3`.
105 fn broadcast_last(self, v: Self::V) -> Self::V;
106 /// The last lane `v3` as a scalar.
107 fn last_lane(self, v: Self::V) -> f64;
108 /// The 4×4 transpose of the rows `a`, `b`, `c`, `d`: result `j` is
109 /// `[a_j, b_j, c_j, d_j]`. Pure data movement, so no value changes.
110 fn transpose4(self, a: Self::V, b: Self::V, c: Self::V, d: Self::V) -> [Self::V; 4];
111
112 /// Eight `f64` lanes for element-wise work: one 512-bit register where the
113 /// CPU has AVX-512 and the kernel opts in ([`Kernel::WIDE`]), two
114 /// four-lane vectors otherwise. Every operation on them is exact per lane,
115 /// so the width never changes a result.
116 type W: Copy;
117
118 /// Eight lanes from an array.
119 fn load8(self, a: &[f64; 8]) -> Self::W;
120 /// Eight lanes into an array.
121 fn store8(self, v: Self::W, a: &mut [f64; 8]);
122 /// All eight lanes `x`.
123 fn splat8(self, x: f64) -> Self::W;
124 /// Lane-wise `a + b` over eight lanes.
125 fn add8(self, a: Self::W, b: Self::W) -> Self::W;
126 /// Lane-wise `a - b` over eight lanes.
127 fn sub8(self, a: Self::W, b: Self::W) -> Self::W;
128 /// Lane-wise `a * b` over eight lanes.
129 fn mul8(self, a: Self::W, b: Self::W) -> Self::W;
130 /// Lane-wise `a / b` over eight lanes.
131 fn div8(self, a: Self::W, b: Self::W) -> Self::W;
132 /// Lane-wise square root over eight lanes.
133 fn sqrt8(self, a: Self::W) -> Self::W;
134 /// Lane-wise `if a > b { a } else { b }` over eight lanes.
135 fn max8(self, a: Self::W, b: Self::W) -> Self::W;
136 /// Lane-wise `if test > 0 { yes } else { no }` over eight lanes (`NaN`
137 /// takes `no`).
138 fn select_positive8(self, test: Self::W, yes: Self::W, no: Self::W) -> Self::W;
139}
140
141/// A unit of work [`dispatch`] can run with the best instruction set available.
142///
143/// `run` receives a [`Simd`] token: vector kernels compute through it, scalar
144/// kernels ignore it and profit from the enabled instruction set through their
145/// plain arithmetic (a `mul_add` becomes the FMA instruction). Implementors mark
146/// [`run`](Kernel::run) `#[inline(always)]` so the body is compiled into the
147/// feature-enabled function rather than called out of it.
148pub trait Kernel {
149 /// What the kernel returns.
150 type Output;
151
152 /// Whether the kernel runs at the AVX-512 level where the CPU has it. A
153 /// kernel opts in when it does its element-wise work in eight lanes
154 /// ([`Simd::W`]) or its plain loops gain from the compiler widening them;
155 /// any other kernel runs at the AVX2 level on such a CPU, as code the
156 /// compiler vectorizes for AVX-512 is not always faster.
157 const WIDE: bool = false;
158
159 /// Execute the kernel with the lanes `simd` provides.
160 fn run<S: Simd>(self, simd: S) -> Self::Output;
161}
162
163/// The instruction set [`dispatch`] selects on this machine.
164#[derive(Debug, Clone, Copy, PartialEq, Eq)]
165pub enum Level {
166 /// Baseline instructions of the build target.
167 Baseline,
168 /// `x86_64` with AVX2 and FMA.
169 Avx2Fma,
170 /// `x86_64` with AVX2, FMA and AVX-512F: kernels that opt in
171 /// ([`Kernel::WIDE`]) get their eight-lane work in one register. Detected
172 /// only in a build by a Rust that supports AVX-512 (1.89 and later).
173 Avx512,
174 /// `aarch64`, whose baseline includes NEON and FMA.
175 Neon,
176}
177
178impl Level {
179 /// Stable lowercase name, for diagnostics and benchmark labels.
180 pub const fn name(self) -> &'static str {
181 match self {
182 Self::Baseline => "baseline",
183 Self::Avx2Fma => "avx2+fma",
184 Self::Avx512 => "avx512",
185 Self::Neon => "neon",
186 }
187 }
188}
189
190/// Cached detection result: 0 = not yet detected, 1 = baseline, 2 = AVX2+FMA,
191/// 3 = AVX-512.
192static DETECTED: AtomicU8 = AtomicU8::new(0);
193
194/// The instruction set [`dispatch`] runs kernels with on this machine.
195pub fn level() -> Level {
196 if cfg!(target_arch = "aarch64") {
197 return Level::Neon;
198 }
199 match DETECTED.load(Ordering::Relaxed) {
200 1 => Level::Baseline,
201 2 => Level::Avx2Fma,
202 3 => Level::Avx512,
203 _ => {
204 let level = detect();
205 let code = match level {
206 Level::Avx512 => 3,
207 Level::Avx2Fma => 2,
208 Level::Baseline | Level::Neon => 1,
209 };
210 DETECTED.store(code, Ordering::Relaxed);
211 level
212 }
213 }
214}
215
216#[cfg(target_arch = "x86_64")]
217fn detect() -> Level {
218 if std::arch::is_x86_feature_detected!("avx2") && std::arch::is_x86_feature_detected!("fma") {
219 if avx512() {
220 Level::Avx512
221 } else {
222 Level::Avx2Fma
223 }
224 } else {
225 Level::Baseline
226 }
227}
228
229/// Whether the CPU has AVX-512F, in a build that can use it.
230#[cfg(all(target_arch = "x86_64", wickra_avx512))]
231fn avx512() -> bool {
232 std::arch::is_x86_feature_detected!("avx512f")
233}
234
235#[cfg(all(target_arch = "x86_64", not(wickra_avx512)))]
236fn avx512() -> bool {
237 false
238}
239
240#[cfg(not(target_arch = "x86_64"))]
241fn detect() -> Level {
242 Level::Baseline
243}
244
245/// Run `kernel` with the best instruction set this CPU offers.
246///
247/// Returns bit-for-bit what [`run_baseline`] returns for the same kernel.
248#[inline]
249pub fn dispatch<K: Kernel>(kernel: K) -> K::Output {
250 run_level(level(), kernel)
251}
252
253/// `kernel` at `level`, which the caller has checked this CPU supports.
254#[inline]
255fn run_level<K: Kernel>(level: Level, kernel: K) -> K::Output {
256 match level {
257 #[cfg(all(target_arch = "x86_64", wickra_avx512))]
258 Level::Avx512 if K::WIDE => avx512::run(kernel),
259 #[cfg(target_arch = "x86_64")]
260 Level::Avx2Fma | Level::Avx512 => avx2::run(kernel),
261 _ => kernel.run(Portable),
262 }
263}
264
265/// Run `kernel` at `level` if this CPU supports it, `None` otherwise -- for
266/// tests that prove a kernel returns the same bits at every level the machine
267/// has, not only the one [`dispatch`] picks.
268#[doc(hidden)]
269pub fn run_at<K: Kernel>(level: Level, kernel: K) -> Option<K::Output> {
270 let best = self::level();
271 let available = match level {
272 Level::Baseline => true,
273 Level::Avx2Fma => matches!(best, Level::Avx2Fma | Level::Avx512),
274 Level::Avx512 | Level::Neon => best == level,
275 };
276 available.then(|| run_level(level, kernel))
277}
278
279/// Run `kernel` in the baseline build with the [`Portable`] lanes, never
280/// dispatching. Exists so tests (in this crate and in its users) can prove a
281/// kernel returns the same bits on every path.
282#[inline(never)]
283pub fn run_baseline<K: Kernel>(kernel: K) -> K::Output {
284 kernel.run(Portable)
285}
286
287#[cfg(test)]
288// Kernels must inline into the dispatching function to be compiled with its
289// features; `#[inline(always)]` on `run` is the documented contract.
290#[allow(clippy::inline_always)]
291mod tests {
292 use super::*;
293
294 /// Three interleaved EMA-style recurrences with `mul_add` plus a plain
295 /// multiply-add the compiler must not fuse: the shape of the scalar kernels.
296 struct Recurrences<'a> {
297 xs: &'a [f64],
298 }
299
300 impl Kernel for Recurrences<'_> {
301 type Output = Vec<f64>;
302
303 #[inline(always)]
304 fn run<S: Simd>(self, _simd: S) -> Vec<f64> {
305 let (mut a, mut b, mut c) = (1.0_f64, 2.0_f64, 3.0_f64);
306 let mut out = Vec::with_capacity(self.xs.len() * 3);
307 for &x in self.xs {
308 a = 0.1_f64.mul_add(x, 0.9 * a);
309 b = 0.3_f64.mul_add(x, 0.7 * b);
310 c = c * 0.95 + x * 0.05;
311 out.extend_from_slice(&[a, b, c - a]);
312 }
313 out
314 }
315 }
316
317 /// Every lane operation over one batch of inputs, the results concatenated.
318 struct AllOps<'a> {
319 a: &'a [f64],
320 b: &'a [f64],
321 }
322
323 impl Kernel for AllOps<'_> {
324 type Output = Vec<f64>;
325 const WIDE: bool = true;
326
327 #[inline(always)]
328 fn run<S: Simd>(self, s: S) -> Vec<f64> {
329 let mut out = Vec::new();
330 for (ca, cb) in self.a.chunks_exact(4).zip(self.b.chunks_exact(4)) {
331 let x = s.load(ca.try_into().unwrap());
332 let y = s.load(cb.try_into().unwrap());
333 let z = s.splat(ca[0]);
334 let mut stored = [0.0; 4];
335 s.store(s.add(x, y), &mut stored);
336 out.extend_from_slice(&stored);
337 for v in [
338 s.sub(x, y),
339 s.mul(x, y),
340 s.div(x, y),
341 s.sqrt(s.abs(x)),
342 s.mul_add(x, y, z),
343 s.max(x, y),
344 s.min(x, y),
345 s.abs(x),
346 s.select_positive(s.sub(x, y), x, y),
347 s.shift1(x),
348 s.shift2(x),
349 s.broadcast_last(x),
350 s.splat(s.last_lane(y)),
351 z,
352 ] {
353 out.extend_from_slice(&s.to_array(v));
354 }
355 for v in s.transpose4(x, y, z, s.sub(x, y)) {
356 out.extend_from_slice(&s.to_array(v));
357 }
358 }
359 for (ca, cb) in self.a.chunks_exact(8).zip(self.b.chunks_exact(8)) {
360 let x = s.load8(ca.try_into().unwrap());
361 let y = s.load8(cb.try_into().unwrap());
362 let z = s.splat8(ca[0]);
363 for v in [
364 s.add8(x, y),
365 s.sub8(x, y),
366 s.mul8(x, y),
367 s.div8(x, y),
368 s.sqrt8(s.max8(x, z)),
369 s.max8(x, y),
370 s.select_positive8(s.sub8(x, y), x, y),
371 z,
372 ] {
373 let mut stored = [0.0; 8];
374 s.store8(v, &mut stored);
375 out.extend_from_slice(&stored);
376 }
377 }
378 out
379 }
380 }
381
382 fn bits(v: &[f64]) -> Vec<u64> {
383 v.iter().map(|x| x.to_bits()).collect()
384 }
385
386 fn series() -> Vec<f64> {
387 (0..4096)
388 .map(|i| (f64::from(i) * 0.731).sin() * 1e3 + f64::from(i % 17))
389 .collect()
390 }
391
392 #[test]
393 fn dispatch_matches_the_baseline_bit_for_bit() {
394 let xs = series();
395 assert_eq!(
396 bits(&dispatch(Recurrences { xs: &xs })),
397 bits(&run_baseline(Recurrences { xs: &xs }))
398 );
399 }
400
401 /// Each lane operation must give the same bits on every implementation,
402 /// the awkward values included: signed zeros, infinities, `NaN` (where
403 /// `max`/`min` must take the second operand), subnormals and a
404 /// cancellation-heavy fused multiply-add.
405 #[test]
406 fn every_lane_operation_matches_the_portable_lanes() {
407 let edge = [
408 0.0,
409 -0.0,
410 f64::INFINITY,
411 f64::NEG_INFINITY,
412 f64::NAN,
413 f64::MIN_POSITIVE / 8.0,
414 1.0 + f64::EPSILON,
415 -1.0,
416 ];
417 let mut a: Vec<f64> = series();
418 let mut b: Vec<f64> = series().iter().rev().map(|x| x * 0.37 - 2.0).collect();
419 for (i, &x) in edge.iter().enumerate() {
420 for (j, &y) in edge.iter().enumerate() {
421 a.extend_from_slice(&[x, y, 1.0 + f64::EPSILON, x]);
422 b.extend_from_slice(&[
423 y,
424 x,
425 1.0 - f64::EPSILON,
426 f64::from(u8::try_from(i * 8 + j).unwrap()),
427 ]);
428 }
429 }
430 let fast = dispatch(AllOps { a: &a, b: &b });
431 let base = run_baseline(AllOps { a: &a, b: &b });
432 assert_eq!(fast.len(), base.len());
433 for (k, (x, y)) in fast.iter().zip(&base).enumerate() {
434 assert!(
435 x.to_bits() == y.to_bits() || (x.is_nan() && y.is_nan()),
436 "lane {k}: {x:e} vs {y:e}"
437 );
438 }
439 }
440
441 #[test]
442 fn portable_lanes_follow_the_documented_rules() {
443 let s = Portable;
444 let x = [1.0, f64::NAN, -0.0, 3.0];
445 let y = [2.0, 5.0, 0.0, f64::NAN];
446 assert_eq!(bits(&s.max(x, y)), bits(&[2.0, 5.0, 0.0, f64::NAN]));
447 assert_eq!(bits(&s.min(x, y)), bits(&[1.0, 5.0, 0.0, f64::NAN]));
448 assert_eq!(s.shift1([1.0, 2.0, 3.0, 4.0]), [0.0, 1.0, 2.0, 3.0]);
449 assert_eq!(s.shift2([1.0, 2.0, 3.0, 4.0]), [0.0, 0.0, 1.0, 2.0]);
450 assert_eq!(s.broadcast_last([1.0, 2.0, 3.0, 4.0]), [4.0; 4]);
451 assert_eq!(
452 bits(&s.abs([-0.0, -2.0, 2.0, -f64::INFINITY])),
453 bits(&[0.0, 2.0, 2.0, f64::INFINITY])
454 );
455 assert_eq!(
456 s.transpose4(
457 [1.0, 2.0, 3.0, 4.0],
458 [5.0, 6.0, 7.0, 8.0],
459 [9.0, 10.0, 11.0, 12.0],
460 [13.0, 14.0, 15.0, 16.0]
461 ),
462 [
463 [1.0, 5.0, 9.0, 13.0],
464 [2.0, 6.0, 10.0, 14.0],
465 [3.0, 7.0, 11.0, 15.0],
466 [4.0, 8.0, 12.0, 16.0]
467 ]
468 );
469 }
470
471 #[test]
472 fn level_is_stable_and_named() {
473 let first = level();
474 assert_eq!(level(), first);
475 assert!(["baseline", "avx2+fma", "avx512", "neon"].contains(&first.name()));
476 assert_eq!(Level::Baseline.name(), "baseline");
477 assert_eq!(Level::Avx2Fma.name(), "avx2+fma");
478 assert_eq!(Level::Avx512.name(), "avx512");
479 assert_eq!(Level::Neon.name(), "neon");
480 }
481
482 #[cfg(target_arch = "x86_64")]
483 #[test]
484 fn detection_agrees_with_the_standard_library() {
485 let both = std::arch::is_x86_feature_detected!("avx2")
486 && std::arch::is_x86_feature_detected!("fma");
487 assert_eq!(matches!(level(), Level::Avx2Fma | Level::Avx512), both);
488 let wide = cfg!(wickra_avx512) && both && avx512();
489 assert_eq!(level() == Level::Avx512, wide);
490 }
491
492 /// Every level the machine has, and only those, runs a kernel -- with the
493 /// bits of the baseline build.
494 #[test]
495 fn every_available_level_matches_the_baseline() {
496 let a: Vec<f64> = series();
497 let b: Vec<f64> = series().iter().map(|x| x * 0.5 + 1.0).collect();
498 let base = run_baseline(AllOps { a: &a, b: &b });
499 let best = level();
500 for at in [Level::Baseline, Level::Avx2Fma, Level::Avx512, Level::Neon] {
501 let ran = run_at(at, AllOps { a: &a, b: &b });
502 let expected = match at {
503 Level::Baseline => true,
504 Level::Avx2Fma => matches!(best, Level::Avx2Fma | Level::Avx512),
505 Level::Avx512 | Level::Neon => best == at,
506 };
507 assert_eq!(ran.is_some(), expected, "{}", at.name());
508 if let Some(out) = ran {
509 assert!(out
510 .iter()
511 .zip(&base)
512 .all(|(x, y)| x.to_bits() == y.to_bits() || (x.is_nan() && y.is_nan())));
513 }
514 }
515 }
516}