use std::sync::atomic::{AtomicU8, Ordering};
mod avx2;
#[clippy::msrv = "1.89"]
mod avx512;
mod portable;
#[cfg(target_arch = "x86_64")]
pub use avx2::Avx2;
#[cfg(all(target_arch = "x86_64", wickra_avx512))]
pub use avx512::Avx512;
pub use portable::Portable;
pub trait Simd: Copy {
type V: Copy;
fn splat(self, x: f64) -> Self::V;
fn load(self, a: &[f64; 4]) -> Self::V;
fn store(self, v: Self::V, a: &mut [f64; 4]);
fn to_array(self, v: Self::V) -> [f64; 4];
fn add(self, a: Self::V, b: Self::V) -> Self::V;
fn sub(self, a: Self::V, b: Self::V) -> Self::V;
fn mul(self, a: Self::V, b: Self::V) -> Self::V;
fn div(self, a: Self::V, b: Self::V) -> Self::V;
fn sqrt(self, a: Self::V) -> Self::V;
fn mul_add(self, a: Self::V, b: Self::V, c: Self::V) -> Self::V;
fn max(self, a: Self::V, b: Self::V) -> Self::V;
fn min(self, a: Self::V, b: Self::V) -> Self::V;
fn abs(self, a: Self::V) -> Self::V;
fn select_positive(self, test: Self::V, yes: Self::V, no: Self::V) -> Self::V;
fn shift1(self, v: Self::V) -> Self::V;
fn shift2(self, v: Self::V) -> Self::V;
fn broadcast_last(self, v: Self::V) -> Self::V;
fn last_lane(self, v: Self::V) -> f64;
fn transpose4(self, a: Self::V, b: Self::V, c: Self::V, d: Self::V) -> [Self::V; 4];
type W: Copy;
fn load8(self, a: &[f64; 8]) -> Self::W;
fn store8(self, v: Self::W, a: &mut [f64; 8]);
fn splat8(self, x: f64) -> Self::W;
fn add8(self, a: Self::W, b: Self::W) -> Self::W;
fn sub8(self, a: Self::W, b: Self::W) -> Self::W;
fn mul8(self, a: Self::W, b: Self::W) -> Self::W;
fn div8(self, a: Self::W, b: Self::W) -> Self::W;
fn sqrt8(self, a: Self::W) -> Self::W;
fn max8(self, a: Self::W, b: Self::W) -> Self::W;
fn select_positive8(self, test: Self::W, yes: Self::W, no: Self::W) -> Self::W;
}
pub trait Kernel {
type Output;
const WIDE: bool = false;
fn run<S: Simd>(self, simd: S) -> Self::Output;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Level {
Baseline,
Avx2Fma,
Avx512,
Neon,
}
impl Level {
pub const fn name(self) -> &'static str {
match self {
Self::Baseline => "baseline",
Self::Avx2Fma => "avx2+fma",
Self::Avx512 => "avx512",
Self::Neon => "neon",
}
}
}
static DETECTED: AtomicU8 = AtomicU8::new(0);
pub fn level() -> Level {
if cfg!(target_arch = "aarch64") {
return Level::Neon;
}
match DETECTED.load(Ordering::Relaxed) {
1 => Level::Baseline,
2 => Level::Avx2Fma,
3 => Level::Avx512,
_ => {
let level = detect();
let code = match level {
Level::Avx512 => 3,
Level::Avx2Fma => 2,
Level::Baseline | Level::Neon => 1,
};
DETECTED.store(code, Ordering::Relaxed);
level
}
}
}
#[cfg(target_arch = "x86_64")]
fn detect() -> Level {
if std::arch::is_x86_feature_detected!("avx2") && std::arch::is_x86_feature_detected!("fma") {
if avx512() {
Level::Avx512
} else {
Level::Avx2Fma
}
} else {
Level::Baseline
}
}
#[cfg(all(target_arch = "x86_64", wickra_avx512))]
fn avx512() -> bool {
std::arch::is_x86_feature_detected!("avx512f")
}
#[cfg(all(target_arch = "x86_64", not(wickra_avx512)))]
fn avx512() -> bool {
false
}
#[cfg(not(target_arch = "x86_64"))]
fn detect() -> Level {
Level::Baseline
}
#[inline]
pub fn dispatch<K: Kernel>(kernel: K) -> K::Output {
run_level(level(), kernel)
}
#[inline]
fn run_level<K: Kernel>(level: Level, kernel: K) -> K::Output {
match level {
#[cfg(all(target_arch = "x86_64", wickra_avx512))]
Level::Avx512 if K::WIDE => avx512::run(kernel),
#[cfg(target_arch = "x86_64")]
Level::Avx2Fma | Level::Avx512 => avx2::run(kernel),
_ => kernel.run(Portable),
}
}
#[doc(hidden)]
pub fn run_at<K: Kernel>(level: Level, kernel: K) -> Option<K::Output> {
let best = self::level();
let available = match level {
Level::Baseline => true,
Level::Avx2Fma => matches!(best, Level::Avx2Fma | Level::Avx512),
Level::Avx512 | Level::Neon => best == level,
};
available.then(|| run_level(level, kernel))
}
#[inline(never)]
pub fn run_baseline<K: Kernel>(kernel: K) -> K::Output {
kernel.run(Portable)
}
#[cfg(test)]
#[allow(clippy::inline_always)]
mod tests {
use super::*;
struct Recurrences<'a> {
xs: &'a [f64],
}
impl Kernel for Recurrences<'_> {
type Output = Vec<f64>;
#[inline(always)]
fn run<S: Simd>(self, _simd: S) -> Vec<f64> {
let (mut a, mut b, mut c) = (1.0_f64, 2.0_f64, 3.0_f64);
let mut out = Vec::with_capacity(self.xs.len() * 3);
for &x in self.xs {
a = 0.1_f64.mul_add(x, 0.9 * a);
b = 0.3_f64.mul_add(x, 0.7 * b);
c = c * 0.95 + x * 0.05;
out.extend_from_slice(&[a, b, c - a]);
}
out
}
}
struct AllOps<'a> {
a: &'a [f64],
b: &'a [f64],
}
impl Kernel for AllOps<'_> {
type Output = Vec<f64>;
const WIDE: bool = true;
#[inline(always)]
fn run<S: Simd>(self, s: S) -> Vec<f64> {
let mut out = Vec::new();
for (ca, cb) in self.a.chunks_exact(4).zip(self.b.chunks_exact(4)) {
let x = s.load(ca.try_into().unwrap());
let y = s.load(cb.try_into().unwrap());
let z = s.splat(ca[0]);
let mut stored = [0.0; 4];
s.store(s.add(x, y), &mut stored);
out.extend_from_slice(&stored);
for v in [
s.sub(x, y),
s.mul(x, y),
s.div(x, y),
s.sqrt(s.abs(x)),
s.mul_add(x, y, z),
s.max(x, y),
s.min(x, y),
s.abs(x),
s.select_positive(s.sub(x, y), x, y),
s.shift1(x),
s.shift2(x),
s.broadcast_last(x),
s.splat(s.last_lane(y)),
z,
] {
out.extend_from_slice(&s.to_array(v));
}
for v in s.transpose4(x, y, z, s.sub(x, y)) {
out.extend_from_slice(&s.to_array(v));
}
}
for (ca, cb) in self.a.chunks_exact(8).zip(self.b.chunks_exact(8)) {
let x = s.load8(ca.try_into().unwrap());
let y = s.load8(cb.try_into().unwrap());
let z = s.splat8(ca[0]);
for v in [
s.add8(x, y),
s.sub8(x, y),
s.mul8(x, y),
s.div8(x, y),
s.sqrt8(s.max8(x, z)),
s.max8(x, y),
s.select_positive8(s.sub8(x, y), x, y),
z,
] {
let mut stored = [0.0; 8];
s.store8(v, &mut stored);
out.extend_from_slice(&stored);
}
}
out
}
}
fn bits(v: &[f64]) -> Vec<u64> {
v.iter().map(|x| x.to_bits()).collect()
}
fn series() -> Vec<f64> {
(0..4096)
.map(|i| (f64::from(i) * 0.731).sin() * 1e3 + f64::from(i % 17))
.collect()
}
#[test]
fn dispatch_matches_the_baseline_bit_for_bit() {
let xs = series();
assert_eq!(
bits(&dispatch(Recurrences { xs: &xs })),
bits(&run_baseline(Recurrences { xs: &xs }))
);
}
#[test]
fn every_lane_operation_matches_the_portable_lanes() {
let edge = [
0.0,
-0.0,
f64::INFINITY,
f64::NEG_INFINITY,
f64::NAN,
f64::MIN_POSITIVE / 8.0,
1.0 + f64::EPSILON,
-1.0,
];
let mut a: Vec<f64> = series();
let mut b: Vec<f64> = series().iter().rev().map(|x| x * 0.37 - 2.0).collect();
for (i, &x) in edge.iter().enumerate() {
for (j, &y) in edge.iter().enumerate() {
a.extend_from_slice(&[x, y, 1.0 + f64::EPSILON, x]);
b.extend_from_slice(&[
y,
x,
1.0 - f64::EPSILON,
f64::from(u8::try_from(i * 8 + j).unwrap()),
]);
}
}
let fast = dispatch(AllOps { a: &a, b: &b });
let base = run_baseline(AllOps { a: &a, b: &b });
assert_eq!(fast.len(), base.len());
for (k, (x, y)) in fast.iter().zip(&base).enumerate() {
assert!(
x.to_bits() == y.to_bits() || (x.is_nan() && y.is_nan()),
"lane {k}: {x:e} vs {y:e}"
);
}
}
#[test]
fn portable_lanes_follow_the_documented_rules() {
let s = Portable;
let x = [1.0, f64::NAN, -0.0, 3.0];
let y = [2.0, 5.0, 0.0, f64::NAN];
assert_eq!(bits(&s.max(x, y)), bits(&[2.0, 5.0, 0.0, f64::NAN]));
assert_eq!(bits(&s.min(x, y)), bits(&[1.0, 5.0, 0.0, f64::NAN]));
assert_eq!(s.shift1([1.0, 2.0, 3.0, 4.0]), [0.0, 1.0, 2.0, 3.0]);
assert_eq!(s.shift2([1.0, 2.0, 3.0, 4.0]), [0.0, 0.0, 1.0, 2.0]);
assert_eq!(s.broadcast_last([1.0, 2.0, 3.0, 4.0]), [4.0; 4]);
assert_eq!(
bits(&s.abs([-0.0, -2.0, 2.0, -f64::INFINITY])),
bits(&[0.0, 2.0, 2.0, f64::INFINITY])
);
assert_eq!(
s.transpose4(
[1.0, 2.0, 3.0, 4.0],
[5.0, 6.0, 7.0, 8.0],
[9.0, 10.0, 11.0, 12.0],
[13.0, 14.0, 15.0, 16.0]
),
[
[1.0, 5.0, 9.0, 13.0],
[2.0, 6.0, 10.0, 14.0],
[3.0, 7.0, 11.0, 15.0],
[4.0, 8.0, 12.0, 16.0]
]
);
}
#[test]
fn level_is_stable_and_named() {
let first = level();
assert_eq!(level(), first);
assert!(["baseline", "avx2+fma", "avx512", "neon"].contains(&first.name()));
assert_eq!(Level::Baseline.name(), "baseline");
assert_eq!(Level::Avx2Fma.name(), "avx2+fma");
assert_eq!(Level::Avx512.name(), "avx512");
assert_eq!(Level::Neon.name(), "neon");
}
#[cfg(target_arch = "x86_64")]
#[test]
fn detection_agrees_with_the_standard_library() {
let both = std::arch::is_x86_feature_detected!("avx2")
&& std::arch::is_x86_feature_detected!("fma");
assert_eq!(matches!(level(), Level::Avx2Fma | Level::Avx512), both);
let wide = cfg!(wickra_avx512) && both && avx512();
assert_eq!(level() == Level::Avx512, wide);
}
#[test]
fn every_available_level_matches_the_baseline() {
let a: Vec<f64> = series();
let b: Vec<f64> = series().iter().map(|x| x * 0.5 + 1.0).collect();
let base = run_baseline(AllOps { a: &a, b: &b });
let best = level();
for at in [Level::Baseline, Level::Avx2Fma, Level::Avx512, Level::Neon] {
let ran = run_at(at, AllOps { a: &a, b: &b });
let expected = match at {
Level::Baseline => true,
Level::Avx2Fma => matches!(best, Level::Avx2Fma | Level::Avx512),
Level::Avx512 | Level::Neon => best == at,
};
assert_eq!(ran.is_some(), expected, "{}", at.name());
if let Some(out) = ran {
assert!(out
.iter()
.zip(&base)
.all(|(x, y)| x.to_bits() == y.to_bits() || (x.is_nan() && y.is_nan())));
}
}
}
}