#![allow(clippy::float_cmp)]
#![allow(clippy::inline_always)]
#![cfg_attr(
not(any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")),
allow(dead_code)
)]
use pulp::Simd;
use crate::dd::DoubleDouble;
use crate::eft::two_prod;
use crate::kernels::{ColumnUpdate, Compensated, LANES, reference};
pub(crate) trait Lanes: Copy {
type V: Copy;
fn splat(self, x: f64) -> Self::V;
fn load(self, a: [f64; LANES]) -> Self::V;
fn store(self, v: Self::V) -> [f64; LANES];
fn add(self, a: Self::V, b: Self::V) -> Self::V;
fn sub(self, a: Self::V, b: Self::V) -> Self::V;
fn mul(self, a: Self::V, b: Self::V) -> Self::V;
fn mul_add(self, a: Self::V, b: Self::V, c: Self::V) -> Self::V;
fn neg(self, a: Self::V) -> Self::V;
fn abs(self, a: Self::V) -> Self::V;
fn all_finite(self, a: Self::V, b: Self::V) -> bool;
}
#[derive(Clone, Copy, Debug)]
pub(crate) struct X4<S: Simd>(S);
impl<S: Simd> X4<S> {
#[inline(always)]
#[cfg_attr(
not(any(target_arch = "x86", target_arch = "x86_64")),
allow(dead_code)
)]
pub(crate) fn new(simd: S) -> Self {
const { assert!(S::F64_LANES == LANES) };
Self(simd)
}
}
impl<S: Simd> Lanes for X4<S> {
type V = S::f64s;
#[inline(always)]
fn splat(self, x: f64) -> Self::V {
self.0.splat_f64s(x)
}
#[inline(always)]
fn load(self, a: [f64; LANES]) -> Self::V {
pulp::cast(a)
}
#[inline(always)]
fn store(self, v: Self::V) -> [f64; LANES] {
pulp::cast(v)
}
#[inline(always)]
fn add(self, a: Self::V, b: Self::V) -> Self::V {
self.0.add_f64s(a, b)
}
#[inline(always)]
fn sub(self, a: Self::V, b: Self::V) -> Self::V {
self.0.sub_f64s(a, b)
}
#[inline(always)]
fn mul(self, a: Self::V, b: Self::V) -> Self::V {
self.0.mul_f64s(a, b)
}
#[inline(always)]
fn mul_add(self, a: Self::V, b: Self::V, c: Self::V) -> Self::V {
self.0.mul_add_f64s(a, b, c)
}
#[inline(always)]
fn neg(self, a: Self::V) -> Self::V {
self.0.neg_f64s(a)
}
#[inline(always)]
fn abs(self, a: Self::V) -> Self::V {
self.0.abs_f64s(a)
}
#[inline(always)]
fn all_finite(self, a: Self::V, b: Self::V) -> bool {
let s = self.0;
let z = s.add_f64s(s.sub_f64s(a, a), s.sub_f64s(b, b));
s.reduce_sum_f64s(z) == 0.0
}
}
#[derive(Clone, Copy, Debug)]
pub(crate) struct X2<S: Simd>(S);
impl<S: Simd> X2<S> {
#[inline(always)]
#[cfg_attr(not(target_arch = "aarch64"), allow(dead_code))]
pub(crate) fn new(simd: S) -> Self {
const { assert!(2 * S::F64_LANES == LANES) };
Self(simd)
}
}
impl<S: Simd> Lanes for X2<S> {
type V = [S::f64s; 2];
#[inline(always)]
fn splat(self, x: f64) -> Self::V {
[self.0.splat_f64s(x); 2]
}
#[inline(always)]
fn load(self, a: [f64; LANES]) -> Self::V {
[pulp::cast([a[0], a[1]]), pulp::cast([a[2], a[3]])]
}
#[inline(always)]
fn store(self, v: Self::V) -> [f64; LANES] {
let (lo, hi): ([f64; 2], [f64; 2]) = (pulp::cast(v[0]), pulp::cast(v[1]));
[lo[0], lo[1], hi[0], hi[1]]
}
#[inline(always)]
fn add(self, a: Self::V, b: Self::V) -> Self::V {
[self.0.add_f64s(a[0], b[0]), self.0.add_f64s(a[1], b[1])]
}
#[inline(always)]
fn sub(self, a: Self::V, b: Self::V) -> Self::V {
[self.0.sub_f64s(a[0], b[0]), self.0.sub_f64s(a[1], b[1])]
}
#[inline(always)]
fn mul(self, a: Self::V, b: Self::V) -> Self::V {
[self.0.mul_f64s(a[0], b[0]), self.0.mul_f64s(a[1], b[1])]
}
#[inline(always)]
fn mul_add(self, a: Self::V, b: Self::V, c: Self::V) -> Self::V {
[
self.0.mul_add_f64s(a[0], b[0], c[0]),
self.0.mul_add_f64s(a[1], b[1], c[1]),
]
}
#[inline(always)]
fn neg(self, a: Self::V) -> Self::V {
[self.0.neg_f64s(a[0]), self.0.neg_f64s(a[1])]
}
#[inline(always)]
fn abs(self, a: Self::V) -> Self::V {
[self.0.abs_f64s(a[0]), self.0.abs_f64s(a[1])]
}
#[inline(always)]
fn all_finite(self, a: Self::V, b: Self::V) -> bool {
let s = self.0;
let z0 = s.add_f64s(s.sub_f64s(a[0], a[0]), s.sub_f64s(b[0], b[0]));
let z1 = s.add_f64s(s.sub_f64s(a[1], a[1]), s.sub_f64s(b[1], b[1]));
s.reduce_sum_f64s(s.add_f64s(z0, z1)) == 0.0
}
}
type Dq<L> = (<L as Lanes>::V, <L as Lanes>::V);
#[inline(always)]
fn two_sum_q<L: Lanes>(l: L, a: L::V, b: L::V) -> (L::V, L::V) {
let s = l.add(a, b);
let a1 = l.sub(s, b);
let b1 = l.sub(s, a1);
let da = l.sub(a, a1);
let db = l.sub(b, b1);
(s, l.add(da, db))
}
#[inline(always)]
fn fast_two_sum_q<L: Lanes>(l: L, a: L::V, b: L::V) -> (L::V, L::V) {
let s = l.add(a, b);
let z = l.sub(s, a);
(s, l.sub(b, z))
}
#[inline(always)]
fn two_prod_q<L: Lanes>(l: L, a: L::V, b: L::V) -> (L::V, L::V) {
let p = l.mul(a, b);
(p, l.mul_add(a, b, l.neg(p)))
}
#[inline(always)]
fn add_q<L: Lanes>(l: L, x: Dq<L>, y: Dq<L>) -> Dq<L> {
let (sh, sl) = two_sum_q(l, x.0, y.0);
let (th, tl) = two_sum_q(l, x.1, y.1);
let c = l.add(sl, th);
let (vh, vl) = fast_two_sum_q(l, sh, c);
let w = l.add(tl, vl);
fast_two_sum_q(l, vh, w)
}
#[inline(always)]
fn mul_q<L: Lanes>(l: L, x: Dq<L>, y: Dq<L>) -> Dq<L> {
let (ch, cl1) = two_prod_q(l, x.0, y.0);
let tl0 = l.mul(x.1, y.1);
let tl1 = l.mul_add(x.0, y.1, tl0);
let cl2 = l.mul_add(x.1, y.0, tl1);
let cl3 = l.add(cl1, cl2);
fast_two_sum_q(l, ch, cl3)
}
#[inline(always)]
fn mul_f64_q<L: Lanes>(l: L, x: Dq<L>, y: L::V) -> Dq<L> {
let (ch, cl1) = two_prod_q(l, x.0, y);
let cl3 = l.mul_add(x.1, y, cl1);
fast_two_sum_q(l, ch, cl3)
}
#[inline(always)]
fn load_dd<L: Lanes>(l: L, c: &[DoubleDouble; LANES]) -> Dq<L> {
(
l.load([c[0].hi(), c[1].hi(), c[2].hi(), c[3].hi()]),
l.load([c[0].lo(), c[1].lo(), c[2].lo(), c[3].lo()]),
)
}
#[inline(always)]
fn store_dd<L: Lanes>(l: L, z: Dq<L>) -> [DoubleDouble; LANES] {
let (h, lo) = (l.store(z.0), l.store(z.1));
[
DoubleDouble::from_raw(h[0], lo[0]),
DoubleDouble::from_raw(h[1], lo[1]),
DoubleDouble::from_raw(h[2], lo[2]),
DoubleDouble::from_raw(h[3], lo[3]),
]
}
#[inline(always)]
fn splat_dd<L: Lanes>(l: L, x: DoubleDouble) -> Dq<L> {
(l.splat(x.hi()), l.splat(x.lo()))
}
#[inline(always)]
pub(crate) fn sum2<L: Lanes>(l: L, x: &[f64]) -> Compensated {
let (chunks, tail) = x.as_chunks::<LANES>();
let zero = l.splat(0.0);
let (mut p, mut s) = (zero, zero);
for c in chunks {
let (pn, q) = two_sum_q(l, p, l.load(*c));
p = pn;
s = l.add(s, l.add(q, zero));
}
let mut acc = Compensated {
p: l.store(p),
s: l.store(s),
e: [0.0; LANES],
};
for (j, &xi) in tail.iter().enumerate() {
acc.push(j, xi, 0.0);
}
acc
}
#[inline(always)]
pub(crate) fn dot2<L: Lanes, const BOUND: bool>(l: L, x: &[f64], y: &[f64]) -> Compensated {
debug_assert_eq!(x.len(), y.len(), "dot2: the lengths differ");
let (cx, tx) = x.as_chunks::<LANES>();
let (cy, ty) = y.as_chunks::<LANES>();
let zero = l.splat(0.0);
let (mut p, mut s, mut e) = (zero, zero, zero);
for (a, b) in cx.iter().zip(cy) {
let (g, r) = two_prod_q(l, l.load(*a), l.load(*b));
let (pn, q) = two_sum_q(l, p, g);
let t = l.add(q, r);
p = pn;
s = l.add(s, t);
if BOUND {
e = l.add(e, l.abs(t));
}
}
let mut acc = Compensated {
p: l.store(p),
s: l.store(s),
e: l.store(e),
};
for (j, (&xi, &yi)) in tx.iter().zip(ty).enumerate() {
let (h, r) = two_prod(xi, yi);
acc.push(j, h, r);
}
acc
}
#[inline(always)]
pub(crate) fn dot<L: Lanes>(l: L, x: &[DoubleDouble], y: &[DoubleDouble]) -> DoubleDouble {
debug_assert_eq!(x.len(), y.len(), "dot: the lengths differ");
let (cx, tx) = x.as_chunks::<LANES>();
let (cy, ty) = y.as_chunks::<LANES>();
let mut a = splat_dd(l, DoubleDouble::ZERO);
for (p, q) in cx.iter().zip(cy) {
a = add_q(l, a, mul_q(l, load_dd(l, p), load_dd(l, q)));
}
if !l.all_finite(a.0, a.1) {
return reference::dot(x, y);
}
let mut acc = store_dd(l, a);
for (j, (&xi, &yi)) in tx.iter().zip(ty).enumerate() {
acc[j] += xi * yi;
}
((acc[0] + acc[1]) + acc[2]) + acc[3]
}
#[inline(always)]
#[cfg_attr(not(feature = "sparse"), allow(dead_code))]
pub(crate) fn dot4<L: Lanes>(l: L, x: &[DoubleDouble], y: &[DoubleDouble]) -> DoubleDouble {
debug_assert_eq!(x.len(), y.len(), "dot4: the lengths differ");
let (cx, tx) = x.as_chunks::<LANES>();
let (cy, ty) = y.as_chunks::<LANES>();
let mut a = splat_dd(l, DoubleDouble::ZERO);
for (p, q) in cx.iter().zip(cy) {
a = add_q(l, a, mul_q(l, load_dd(l, p), load_dd(l, q)));
}
if !l.all_finite(a.0, a.1) {
return reference::dot4(x, y);
}
let acc = store_dd(l, a);
let mut t = DoubleDouble::ZERO;
for (&p, &q) in tx.iter().zip(ty) {
t += p * q;
}
(acc[0] + acc[1]) + (acc[2] + acc[3]) + t
}
#[inline(always)]
pub(crate) fn dot_mixed<L: Lanes>(l: L, x: &[DoubleDouble], y: &[f64]) -> DoubleDouble {
debug_assert_eq!(x.len(), y.len(), "dot_mixed: the lengths differ");
let (cx, tx) = x.as_chunks::<LANES>();
let (cy, ty) = y.as_chunks::<LANES>();
let mut a = splat_dd(l, DoubleDouble::ZERO);
for (p, q) in cx.iter().zip(cy) {
a = add_q(l, a, mul_f64_q(l, load_dd(l, p), l.load(*q)));
}
if !l.all_finite(a.0, a.1) {
return reference::dot_mixed(x, y);
}
let mut acc = store_dd(l, a);
for (j, (&xi, &yi)) in tx.iter().zip(ty).enumerate() {
acc[j] += xi * yi;
}
((acc[0] + acc[1]) + acc[2]) + acc[3]
}
#[inline(always)]
pub(crate) fn axpy<L: Lanes>(
l: L,
alpha: DoubleDouble,
x: &[DoubleDouble],
y: &mut [DoubleDouble],
) {
debug_assert_eq!(x.len(), y.len(), "axpy: the lengths differ");
let (cx, tx) = x.as_chunks::<LANES>();
let (cy, ty) = y.as_chunks_mut::<LANES>();
let al = splat_dd(l, alpha);
for (p, q) in cx.iter().zip(cy) {
let z = add_q(l, load_dd(l, q), mul_q(l, al, load_dd(l, p)));
if l.all_finite(z.0, z.1) {
*q = store_dd(l, z);
} else {
reference::axpy(alpha, p, q);
}
}
reference::axpy(alpha, tx, ty);
}
impl<L: Lanes> ColumnUpdate for L {
#[inline(always)]
fn column_sub(self, c: &mut [DoubleDouble], a: &[DoubleDouble], t: DoubleDouble) {
debug_assert_eq!(c.len(), a.len(), "column_sub: the lengths differ");
let l = self;
let (cc, tc) = c.as_chunks_mut::<LANES>();
let (ca, ta) = a.as_chunks::<LANES>();
let tq = splat_dd(l, t);
for (q, p) in cc.iter_mut().zip(ca) {
let prod = mul_q(l, load_dd(l, p), tq);
let z = add_q(l, load_dd(l, q), (l.neg(prod.0), l.neg(prod.1)));
if l.all_finite(z.0, z.1) {
*q = store_dd(l, z);
} else {
reference::Reference.column_sub(q, p, t);
}
}
reference::Reference.column_sub(tc, ta, t);
}
}
#[cfg(test)]
mod tests {
use super::*;
fn lanes_follow_f64<L: Lanes>(l: L) {
let a = [1.0 + f64::EPSILON, -3.5e200, 1e-310, f64::INFINITY];
let b = [1.0 - f64::EPSILON / 2.0, 2.25e150, -7.0, 1.0];
let c = [-1.0, f64::MAX, 3e-320, f64::NAN];
let (va, vb, vc) = (l.load(a), l.load(b), l.load(c));
let same = |v: L::V, f: &dyn Fn(usize) -> f64| {
for (k, got) in l.store(v).into_iter().enumerate() {
let want = f(k);
assert!(
got.to_bits() == want.to_bits() || (got.is_nan() && want.is_nan()),
"lane {k}: {got:e} != {want:e}"
);
}
};
same(va, &|k| a[k]);
same(l.splat(0.1), &|_| 0.1);
same(l.add(va, vb), &|k| a[k] + b[k]);
same(l.sub(va, vb), &|k| a[k] - b[k]);
same(l.mul(va, vb), &|k| a[k] * b[k]);
same(l.mul_add(va, vb, vc), &|k| a[k].mul_add(b[k], c[k]));
same(l.neg(vc), &|k| -c[k]);
same(l.abs(vb), &|k| b[k].abs());
let fused = l.store(l.mul_add(va, vb, l.splat(-1.0)))[0];
assert_eq!(fused, a[0].mul_add(b[0], -1.0));
assert_ne!(fused, a[0] * b[0] - 1.0);
assert!(l.all_finite(l.splat(1.0), l.load([0.0, -0.0, f64::MAX, 5e-324])));
for bad in [f64::INFINITY, f64::NEG_INFINITY, f64::NAN] {
for k in 0..LANES {
let mut v = [1.0; LANES];
v[k] = bad;
assert!(!l.all_finite(l.load(v), l.splat(0.0)));
assert!(!l.all_finite(l.splat(0.0), l.load(v)));
}
}
}
#[test]
fn the_native_backend_follows_f64() {
#[cfg(target_arch = "aarch64")]
if let Some(token) = pulp::aarch64::Neon::try_new() {
token.vectorize(|| lanes_follow_f64(X2::new(token)));
}
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
if let Some(token) = pulp::x86::V3::try_new() {
token.vectorize(|| lanes_follow_f64(X4::new(token)));
}
}
}