use crate::dd::DoubleDouble;
use crate::eft::two_sum;
pub(crate) const LANES: usize = 4;
pub(crate) const BLOCK_ROWS: usize = 128;
pub(crate) const BLOCK_K: usize = 64;
const U: f64 = 0.5 * f64::EPSILON;
#[derive(Clone, Copy, Default)]
pub(crate) struct Compensated {
pub(crate) p: [f64; LANES],
pub(crate) s: [f64; LANES],
pub(crate) e: [f64; LANES],
}
impl Compensated {
#[inline]
pub(crate) fn push(&mut self, i: usize, h: f64, r: f64) {
let k = i % LANES;
let (p, q) = two_sum(self.p[k], h);
let t = q + r;
self.p[k] = p;
self.s[k] += t;
self.e[k] += t.abs();
}
#[inline]
pub(crate) fn combine(&self) -> (f64, f64, f64) {
let (mut p, mut s, mut e) = (self.p[0], self.s[0], self.e[0]);
for k in 1..LANES {
let (pn, q) = two_sum(p, self.p[k]);
p = pn;
s += self.s[k] + q;
e += self.e[k] + q.abs();
}
(p, s, e)
}
#[inline]
pub(crate) fn result(&self) -> DoubleDouble {
let (p, s, _) = self.combine();
DoubleDouble::from_sum(p, s)
}
#[inline]
pub(crate) fn result_with_bound(&self, n: usize) -> (DoubleDouble, f64) {
const UNDERFLOW: f64 = 6.0 * f64::MIN_POSITIVE;
const INFLATE: f64 = 1.0 + 4.0 * f64::EPSILON;
const MAX_TERMS: u64 = 1 << 52;
let (p, s, e) = self.combine();
let (_, r_lo) = two_sum(p, s);
let r = DoubleDouble::from_sum(p, s);
if n == 0 {
return (r, 0.0);
}
let terms = u64::try_from(n).map_or(u64::MAX, |n| n.saturating_add(3));
let finite =
p.is_finite() && s.is_finite() && e.is_finite() && r.is_finite() && r_lo.is_finite();
if terms >= MAX_TERMS || !finite {
return (r, f64::INFINITY);
}
#[allow(clippy::cast_precision_loss)] let a = terms as f64 * U;
let delta = a / (1.0 - 2.0 * a);
(r, (delta * e + UNDERFLOW) * INFLATE)
}
}
pub(crate) trait ColumnUpdate: Copy {
fn column_sub(self, c: &mut [DoubleDouble], a: &[DoubleDouble], t: DoubleDouble);
}
#[inline(always)] #[allow(clippy::inline_always)]
pub(crate) fn gemm_blocked<U: ColumnUpdate>(
update: U,
m: usize,
k: usize,
a: &[DoubleDouble],
c: &mut [DoubleDouble],
t: impl Fn(usize, usize) -> DoubleDouble,
) {
debug_assert_eq!(a.len(), m * k, "gemm_blocked: A is not m × k");
debug_assert!(m == 0 || c.len().is_multiple_of(m), "gemm_blocked: C");
if m == 0 {
return;
}
let mut i0 = 0;
while i0 < m {
let i1 = (i0 + BLOCK_ROWS).min(m);
let mut l0 = 0;
while l0 < k {
let l1 = (l0 + BLOCK_K).min(k);
for (j, cj) in c.chunks_exact_mut(m).enumerate() {
let cj = &mut cj[i0..i1];
for l in l0..l1 {
update.column_sub(cj, &a[l * m + i0..l * m + i1], t(j, l));
}
}
l0 = l1;
}
i0 = i1;
}
}
pub(crate) mod reference {
use super::{ColumnUpdate, Compensated, LANES, gemm_blocked};
use crate::dd::DoubleDouble;
use crate::eft::two_prod;
#[derive(Clone, Copy, Debug)]
pub(crate) struct Reference;
impl ColumnUpdate for Reference {
#[inline]
fn column_sub(self, c: &mut [DoubleDouble], a: &[DoubleDouble], t: DoubleDouble) {
debug_assert_eq!(c.len(), a.len(), "column_sub: the lengths differ");
for (ci, &ai) in c.iter_mut().zip(a) {
*ci -= ai * t;
}
}
}
pub(crate) fn sum2(x: &[f64]) -> Compensated {
let mut acc = Compensated::default();
for (i, &xi) in x.iter().enumerate() {
acc.push(i, xi, 0.0);
}
acc
}
pub(crate) fn dot2(x: &[f64], y: &[f64]) -> Compensated {
let mut acc = Compensated::default();
for (i, (&xi, &yi)) in x.iter().zip(y).enumerate() {
let (h, r) = two_prod(xi, yi);
acc.push(i, h, r);
}
acc
}
pub(crate) fn dot(x: &[DoubleDouble], y: &[DoubleDouble]) -> DoubleDouble {
let mut acc = [DoubleDouble::ZERO; LANES];
for (i, (&xi, &yi)) in x.iter().zip(y).enumerate() {
acc[i % LANES] += xi * yi;
}
((acc[0] + acc[1]) + acc[2]) + acc[3]
}
#[cfg_attr(not(feature = "sparse"), allow(dead_code))]
pub(crate) fn dot4(x: &[DoubleDouble], y: &[DoubleDouble]) -> DoubleDouble {
let mut acc = [DoubleDouble::ZERO; LANES];
let (cx, tx) = x.as_chunks::<LANES>();
let (cy, ty) = y.as_chunks::<LANES>();
for (p, q) in cx.iter().zip(cy) {
for k in 0..LANES {
acc[k] += p[k] * q[k];
}
}
let mut t = DoubleDouble::ZERO;
for (&p, &q) in tx.iter().zip(ty) {
t += p * q;
}
(acc[0] + acc[1]) + (acc[2] + acc[3]) + t
}
pub(crate) fn dot_mixed(x: &[DoubleDouble], y: &[f64]) -> DoubleDouble {
let mut acc = [DoubleDouble::ZERO; LANES];
for (i, (&xi, &yi)) in x.iter().zip(y).enumerate() {
acc[i % LANES] += xi * yi;
}
((acc[0] + acc[1]) + acc[2]) + acc[3]
}
pub(crate) fn axpy(alpha: DoubleDouble, x: &[DoubleDouble], y: &mut [DoubleDouble]) {
for (yi, &xi) in y.iter_mut().zip(x) {
*yi += alpha * xi;
}
}
pub(crate) fn gemm_sub(
m: usize,
k: usize,
a: &[DoubleDouble],
b: &[DoubleDouble],
c: &mut [DoubleDouble],
) {
gemm_blocked(Reference, m, k, a, c, |j, l| b[l + j * k]);
}
}
macro_rules! dispatch {
(|$lanes:ident| $simd:expr) => {
#[cfg(all(feature = "simd", target_arch = "aarch64"))]
if let Some(token) = pulp::aarch64::Neon::try_new() {
let $lanes = $crate::simd::X2::new(token);
return pulp::Simd::vectorize(
token,
#[inline(always)]
|| $simd,
);
}
#[cfg(all(feature = "simd", any(target_arch = "x86", target_arch = "x86_64")))]
if let Some(token) = pulp::x86::V3::try_new() {
let $lanes = $crate::simd::X4::new(token);
return pulp::Simd::vectorize(
token,
#[inline(always)]
|| $simd,
);
}
};
}
#[cfg(feature = "sparse")]
pub(crate) use dispatch;
#[must_use]
pub fn sum2(x: &[f64]) -> DoubleDouble {
dispatch!(|lanes| crate::simd::sum2(lanes, x).result());
reference::sum2(x).result()
}
#[must_use]
pub fn dot2(x: &[f64], y: &[f64]) -> DoubleDouble {
assert_eq!(x.len(), y.len(), "dot2: the lengths differ");
dispatch!(|lanes| crate::simd::dot2::<_, false>(lanes, x, y).result());
reference::dot2(x, y).result()
}
#[must_use]
pub fn dot2_err(x: &[f64], y: &[f64]) -> (DoubleDouble, f64) {
assert_eq!(x.len(), y.len(), "dot2_err: the lengths differ");
dispatch!(|lanes| crate::simd::dot2::<_, true>(lanes, x, y).result_with_bound(x.len()));
reference::dot2(x, y).result_with_bound(x.len())
}
#[must_use]
pub fn dot(x: &[DoubleDouble], y: &[DoubleDouble]) -> DoubleDouble {
assert_eq!(x.len(), y.len(), "dot: the lengths differ");
dispatch!(|lanes| crate::simd::dot(lanes, x, y));
reference::dot(x, y)
}
#[must_use]
pub fn dot_mixed(x: &[DoubleDouble], y: &[f64]) -> DoubleDouble {
assert_eq!(x.len(), y.len(), "dot_mixed: the lengths differ");
dispatch!(|lanes| crate::simd::dot_mixed(lanes, x, y));
reference::dot_mixed(x, y)
}
pub fn axpy(alpha: DoubleDouble, x: &[DoubleDouble], y: &mut [DoubleDouble]) {
assert_eq!(x.len(), y.len(), "axpy: the lengths differ");
dispatch!(|lanes| crate::simd::axpy(lanes, alpha, x, y));
reference::axpy(alpha, x, y);
}
pub fn gemm_sub(
m: usize,
k: usize,
n: usize,
a: &[DoubleDouble],
b: &[DoubleDouble],
c: &mut [DoubleDouble],
) {
assert_eq!(a.len(), m * k, "gemm_sub: A is not m × k");
assert_eq!(b.len(), k * n, "gemm_sub: B is not k × n");
assert_eq!(c.len(), m * n, "gemm_sub: C is not m × n");
dispatch!(|lanes| gemm_blocked(lanes, m, k, a, c, |j, l| b[l + j * k]));
reference::gemm_sub(m, k, a, b, c);
}
#[must_use]
pub fn backend() -> &'static str {
#[cfg(all(feature = "simd", target_arch = "aarch64"))]
if pulp::aarch64::Neon::try_new().is_some() {
return "neon";
}
#[cfg(all(feature = "simd", any(target_arch = "x86", target_arch = "x86_64")))]
if pulp::x86::V3::try_new().is_some() {
return "avx2";
}
"scalar"
}
#[cfg(test)]
#[allow(clippy::disallowed_methods)] #[allow(clippy::float_cmp)] mod tests {
use super::*;
type Dd = DoubleDouble;
#[test]
fn empty_inputs() {
assert_eq!(sum2(&[]), Dd::ZERO);
assert_eq!(dot2(&[], &[]), Dd::ZERO);
assert_eq!(dot2_err(&[], &[]), (Dd::ZERO, 0.0));
assert_eq!(dot(&[], &[]), Dd::ZERO);
assert_eq!(dot_mixed(&[], &[]), Dd::ZERO);
axpy(Dd::ONE, &[], &mut []);
gemm_sub(0, 0, 0, &[], &[], &mut []);
gemm_sub(0, 2, 3, &[], &[Dd::ONE; 6], &mut []);
}
#[test]
fn backend_names_the_dispatch() {
#[cfg(not(feature = "simd"))]
let expected = "scalar";
#[cfg(all(feature = "simd", target_arch = "aarch64"))]
let expected = "neon";
#[cfg(all(feature = "simd", any(target_arch = "x86", target_arch = "x86_64")))]
let expected = if std::arch::is_x86_feature_detected!("avx2")
&& std::arch::is_x86_feature_detected!("fma")
{
"avx2"
} else {
"scalar"
};
#[cfg(all(
feature = "simd",
not(any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64"))
))]
let expected = "scalar";
assert_eq!(backend(), expected);
}
#[test]
fn dot2_err_refuses_when_the_two_sum_of_c4_overflows() {
use dashu_ratio::RBig;
let lane0 = [
0x7fef_ffff_ffff_fffe_u64,
0x7c90_0000_0000_0001,
0xfc8f_ffff_ffff_fffe,
0xfc8f_ffff_ffff_fffc,
0xf970_0000_0000_0000,
];
let mut x = [0.0; 17];
for (k, bits) in lane0.iter().enumerate() {
x[4 * k] = f64::from_bits(*bits);
}
let y = [1.0; 17];
let exact = x.iter().fold(RBig::from(0), |acc, &v| {
acc + RBig::try_from(v).expect("finite")
});
let reference = reference::dot2(&x, &y).result_with_bound(17);
let dispatched = dot2_err(&x, &y);
for (r, beta) in [reference, dispatched] {
assert_eq!((r.hi().to_bits(), r.lo()), (0x7fef_ffff_ffff_fffe, 0.0));
let err = RBig::try_from(r.hi()).expect("finite") - exact.clone();
let err = if err < RBig::from(0) { -err } else { err };
assert!(err > RBig::from(0));
assert!(
beta == f64::INFINITY || err <= RBig::try_from(beta).expect("finite"),
"K2 fails: |xᵀy − r| = {:e} > β = {beta:e}",
err.to_f64().value()
);
assert_eq!(beta, f64::INFINITY);
}
assert_eq!(dot2(&x, &y), dispatched.0);
}
#[test]
fn k1_near_omega() {
let mut state = 0x2545_f491_4f6c_dd1d_u64;
let mut next = move || {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
state
};
for n in 1..=64 {
for _ in 0..50 {
let x: Vec<f64> = (0..n)
.map(|_| {
let sign = if next() & 1 == 0 { 1.0 } else { -1.0 };
let j = f64::from(u8::try_from(next() % 4).expect("small"));
sign * match next() % 4 {
0 => f64::from_bits(f64::MAX.to_bits() - next() % 4),
1 => 2f64.powi(970) + j * 2f64.powi(918),
2 => 2f64.powi(970) - j * 2f64.powi(918),
_ => 0.0,
}
})
.collect();
let y = vec![1.0; n];
assert!(
same_dd(sum2(&x), reference::sum2(&x).result()),
"sum2 {x:?}"
);
assert!(
same_dd(dot2(&x, &y), reference::dot2(&x, &y).result()),
"dot2 {x:?}"
);
let (r, beta) = dot2_err(&x, &y);
let (r0, beta0) = reference::dot2(&x, &y).result_with_bound(n);
assert!(same_dd(r, r0) && same(beta, beta0), "dot2_err {x:?}");
}
}
}
#[test]
fn sum2_survives_cancellation() {
let x = [1e100, 1.0, -1e100, 2.0];
assert_eq!(x.iter().sum::<f64>(), 2.0);
assert_eq!(sum2(&x), Dd::from_f64(3.0));
let tiny = 2f64.powi(-60);
assert_eq!(sum2(&[1.0, tiny, -1.0]), Dd::from_f64(tiny));
assert_eq!(sum2(&[1.0, tiny]), Dd::from_parts(1.0, tiny));
let y: Vec<f64> = (1..=10).map(f64::from).collect();
assert_eq!(sum2(&y), Dd::from_f64(55.0));
}
#[test]
fn dot2_survives_cancellation() {
let big = 2f64.powi(60);
let x = [big, 1.0 + f64::EPSILON, -big];
let y = [big, 1.0 + f64::EPSILON, big];
assert_eq!(x.iter().zip(&y).map(|(a, b)| a * b).sum::<f64>(), 0.0);
let exact = Dd::from_product(x[1], y[1]);
let r = dot2(&x, &y);
assert_eq!(r.hi(), exact.hi());
let (r_err, beta) = dot2_err(&x, &y);
assert_eq!(r_err, r);
assert!(beta.is_finite() && beta < 1e-10, "{beta:e}");
assert!((r - exact).abs().hi() <= beta);
}
#[test]
fn dot2_err_reports_failed_hypotheses() {
let (r, beta) = dot2_err(&[1e200, 1.0], &[1e200, 1.0]);
assert!(r.check_invariants().is_ok());
assert_eq!(beta, f64::INFINITY);
let (r, beta) = dot2_err(&[f64::NAN], &[1.0]);
assert!(r.is_nan());
assert_eq!(beta, f64::INFINITY);
let (r, beta) = dot2_err(&[1.0, 2.0], &[3.0, 4.0]);
assert_eq!(r, Dd::from_f64(11.0));
assert!(beta > 0.0 && beta < 1e-300);
}
#[test]
fn dd_kernels_follow_their_definitions() {
let x: Vec<Dd> = (1..=7).map(|i| Dd::ONE / f64::from(i)).collect();
let y: Vec<Dd> = (1..=7).map(|i| Dd::from_f64(f64::from(i)).sqrt()).collect();
let yf: Vec<f64> = y.iter().map(|v| v.hi()).collect();
let lanes = |term: &dyn Fn(usize) -> Dd| {
let mut a = [Dd::ZERO; 4];
for i in 0..7 {
a[i % 4] += term(i);
}
((a[0] + a[1]) + a[2]) + a[3]
};
assert_eq!(reference::dot(&x, &y), lanes(&|i| x[i] * y[i]));
assert_eq!(dot(&x, &y), lanes(&|i| x[i] * y[i]));
assert_eq!(dot_mixed(&x, &yf), lanes(&|i| x[i] * yf[i]));
let mut a = [Dd::ZERO; 4];
for i in 0..4 {
a[i] += x[i] * y[i];
}
let mut t = Dd::ZERO;
for i in 4..7 {
t += x[i] * y[i];
}
assert_eq!(reference::dot4(&x, &y), ((a[0] + a[1]) + (a[2] + a[3])) + t);
let alpha = Dd::ONE / 7.0;
let mut z = y.clone();
axpy(alpha, &x, &mut z);
for i in 0..7 {
assert_eq!(z[i], y[i] + alpha * x[i]);
}
let a = &x[..6];
let b = &y[..6];
let mut c = vec![Dd::ONE; 4];
gemm_sub(2, 3, 2, a, b, &mut c);
for j in 0..2 {
for i in 0..2 {
let mut want = Dd::ONE;
for l in 0..3 {
want -= a[i + 2 * l] * b[l + 3 * j];
}
assert_eq!(c[i + 2 * j], want);
}
}
}
#[test]
#[should_panic(expected = "the lengths differ")]
fn length_mismatch_panics() {
let _ = dot2(&[1.0], &[]);
}
struct Rng(u64);
impl Rng {
fn next(&mut self) -> u64 {
self.0 ^= self.0 >> 12;
self.0 ^= self.0 << 25;
self.0 ^= self.0 >> 27;
self.0.wrapping_mul(0x2545_f491_4f6c_dd1d)
}
fn scaled(&mut self, span: i32) -> f64 {
#[allow(clippy::cast_precision_loss)] let m = 1.0 + (self.next() >> 11) as f64 / (1_u64 << 53) as f64;
let e = i32::try_from(self.next() % (2 * u64::from(span.unsigned_abs()) + 1))
.expect("small")
- span;
let v = m * 2f64.powi(e);
if self.next() & 1 == 0 { v } else { -v }
}
fn special(&mut self) -> f64 {
const SPECIAL: [f64; 10] = [
0.0,
-0.0,
f64::INFINITY,
f64::NEG_INFINITY,
f64::NAN,
f64::MAX,
f64::MIN,
f64::MIN_POSITIVE,
5e-324,
-2.5e-310,
];
SPECIAL[usize::try_from(self.next() % 10).expect("small")]
}
fn f64(&mut self, kind: u8) -> f64 {
match kind {
0 => self.scaled(30),
1 => self.scaled(600),
2 => {
if self.next().is_multiple_of(8) {
self.special()
} else {
f64::from_bits(self.next())
}
}
_ => {
if self.next().is_multiple_of(16) {
self.special()
} else {
self.scaled(30)
}
}
}
}
fn dd(&mut self, kind: u8) -> Dd {
let h = self.f64(kind);
let l = self.f64(0) * h * 2f64.powi(-84);
Dd::from_sum(h, l)
}
fn f64s(&mut self, kind: u8, n: usize) -> Vec<f64> {
(0..n).map(|_| self.f64(kind)).collect()
}
fn dds(&mut self, kind: u8, n: usize) -> Vec<Dd> {
(0..n).map(|_| self.dd(kind)).collect()
}
}
fn same(a: f64, b: f64) -> bool {
a.to_bits() == b.to_bits() || (a.is_nan() && b.is_nan())
}
fn same_dd(a: Dd, b: Dd) -> bool {
same(a.hi(), b.hi()) && same(a.lo(), b.lo())
}
fn same_dds(a: &[Dd], b: &[Dd]) -> bool {
a.len() == b.len() && a.iter().zip(b).all(|(&p, &q)| same_dd(p, q))
}
fn dot4(x: &[Dd], y: &[Dd]) -> Dd {
dispatch!(|lanes| crate::simd::dot4(lanes, x, y));
reference::dot4(x, y)
}
fn k1_vectors(kind: u8, n: usize, seed: u64) -> Result<(), String> {
let mut rng = Rng(seed | 1);
let (x, y) = (rng.f64s(kind, n), rng.f64s(kind, n));
let (p, q) = (rng.dds(kind, n), rng.dds(kind, n));
let alpha = rng.dd(kind);
let fail = |what: &str| Err(format!("{what}: kind {kind}, n = {n}, seed {seed}"));
if !same_dd(sum2(&x), reference::sum2(&x).result()) {
return fail("sum2");
}
if !same_dd(dot2(&x, &y), reference::dot2(&x, &y).result()) {
return fail("dot2");
}
let (r, beta) = dot2_err(&x, &y);
let (r0, beta0) = reference::dot2(&x, &y).result_with_bound(n);
if !same_dd(r, r0) || !same(beta, beta0) {
return fail("dot2_err");
}
if !same_dd(dot(&p, &q), reference::dot(&p, &q)) {
return fail("dot");
}
if !same_dd(dot_mixed(&p, &x), reference::dot_mixed(&p, &x)) {
return fail("dot_mixed");
}
if !same_dd(dot4(&p, &q), reference::dot4(&p, &q)) {
return fail("dot4");
}
let (mut z, mut z0) = (q.clone(), q.clone());
axpy(alpha, &p, &mut z);
reference::axpy(alpha, &p, &mut z0);
if !same_dds(&z, &z0) {
return fail("axpy");
}
for v in &z {
v.check_invariants()
.map_err(|e| format!("axpy: {e} (kind {kind}, n = {n}, seed {seed})"))?;
}
Ok(())
}
fn k1_gemm(kind: u8, m: usize, k: usize, n: usize, seed: u64) -> Result<(), String> {
let mut rng = Rng(seed | 1);
let a = rng.dds(kind, m * k);
let b = rng.dds(kind, k * n);
let c = rng.dds(kind, m * n);
let (mut c1, mut c2, mut c3) = (c.clone(), c.clone(), c);
gemm_sub(m, k, n, &a, &b, &mut c1);
reference::gemm_sub(m, k, &a, &b, &mut c2);
for j in 0..n {
for l in 0..k {
for i in 0..m {
c3[i + j * m] -= a[i + l * m] * b[l + j * k];
}
}
}
if same_dds(&c1, &c2) && same_dds(&c2, &c3) {
Ok(())
} else {
Err(format!(
"gemm_sub: kind {kind}, {m} × {k} × {n}, seed {seed}"
))
}
}
#[test]
fn k1_every_length_and_kind() {
for kind in 0..4_u8 {
for n in (0..=67).chain([255, 256, 1000, 4099, 65_539]) {
let seeds = if n <= 67 { 6 } else { 2 };
for s in 0..seeds {
let seed = 0x9e37_79b9_7f4a_7c15_u64
.wrapping_mul(u64::from(kind) + 1)
.wrapping_add((n as u64) << 8)
.wrapping_add(s);
k1_vectors(kind, n, seed).unwrap();
}
}
}
}
#[test]
fn k1_gemm_shapes() {
let shapes = [
(0, 3, 2),
(3, 0, 2),
(3, 2, 0),
(1, 1, 1),
(3, 2, 5),
(4, 4, 4),
(7, 5, 3),
(67, 9, 4),
(BLOCK_ROWS + 5, 3, 2),
(9, BLOCK_K + 7, 2),
(2 * BLOCK_ROWS + 3, BLOCK_K + 1, 3),
];
for kind in 0..4_u8 {
for (s, &(m, k, n)) in shapes.iter().enumerate() {
k1_gemm(kind, m, k, n, 77 + 1000 * u64::from(kind) + s as u64).unwrap();
}
}
}
#[test]
fn k1_exceptional_values_in_each_lane() {
let bad = [
Dd::INFINITY,
-Dd::INFINITY,
Dd::NAN,
Dd::from_f64(f64::MAX),
Dd::from_f64(1e200),
];
for (b, &v) in bad.iter().enumerate() {
for at in 0..12 {
let mut rng = Rng(1 + at as u64 + 100 * b as u64);
let mut p = rng.dds(0, 13);
let q = rng.dds(0, 13);
p[at] = v;
let big = Dd::from_f64(1e200);
for alpha in [Dd::ONE, big, Dd::ZERO] {
let (mut z, mut z0) = (q.clone(), q.clone());
axpy(alpha, &p, &mut z);
reference::axpy(alpha, &p, &mut z0);
assert!(same_dds(&z, &z0), "axpy, value {b} at {at}");
let (mut c, mut c0) = (q.clone(), q.clone());
gemm_sub(13, 1, 1, &p, &[alpha], &mut c);
reference::gemm_sub(13, 1, &p, &[alpha], &mut c0);
assert!(same_dds(&c, &c0), "gemm_sub, value {b} at {at}");
}
assert!(same_dd(dot(&p, &q), reference::dot(&p, &q)));
assert!(same_dd(dot(&p, &p), reference::dot(&p, &p)));
assert!(same_dd(dot4(&p, &q), reference::dot4(&p, &q)));
let y: Vec<f64> = q.iter().map(|d| d.hi() * 1e150).collect();
assert!(same_dd(dot_mixed(&p, &y), reference::dot_mixed(&p, &y)));
let x: Vec<f64> = p.iter().map(|d| d.hi()).collect();
assert!(same_dd(sum2(&x), reference::sum2(&x).result()));
assert!(same_dd(dot2(&x, &y), reference::dot2(&x, &y).result()));
let (r, beta) = dot2_err(&x, &y);
let (r0, beta0) = reference::dot2(&x, &y).result_with_bound(13);
assert!(same_dd(r, r0) && same(beta, beta0));
}
}
}
proptest::proptest! {
#![proptest_config(proptest::prelude::ProptestConfig::with_cases(2000))]
#[test]
fn k1_simd_equals_scalar(kind in 0_u8..4, n in 0_usize..=67, seed in proptest::prelude::any::<u64>()) {
proptest::prop_assert_eq!(k1_vectors(kind, n, seed), Ok(()));
}
#[test]
fn k1_gemm_simd_equals_scalar(
kind in 0_u8..4,
m in 0_usize..40,
k in 0_usize..7,
n in 0_usize..5,
seed in proptest::prelude::any::<u64>(),
) {
proptest::prop_assert_eq!(k1_gemm(kind, m, k, n, seed), Ok(()));
}
}
}