use std::error::Error as _;
use super::{default_faer_limit, LowerContractionError};
use tenferro_tensor::{Error as TensorError, ErrorKind};
fn has_source<T: std::error::Error + 'static>(error: &TensorError) -> bool {
error
.source()
.and_then(|source| source.downcast_ref::<T>())
.is_some()
}
#[test]
fn cpueinsum_workspace_is_a_typed_runtime_state_failure() {
let error = cpueinsum::Error::Workspace {
needed: 16,
have: 8,
}
.into_crate_error("dot_general");
assert_eq!(error.kind(), ErrorKind::RuntimeState);
assert!(has_source::<cpueinsum::Error>(&error));
}
#[test]
fn cpueinsum_declined_route_keeps_its_typed_source() {
let declined = cpueinsum::Error::Contract {
step: 0,
source: cpueinsum::tcontract::Error::Unsupported(
cpueinsum::tcontract::api::Unsupported::Reason("test decline"),
),
};
let error = declined.into_crate_error("dot_general");
assert_eq!(error.kind(), ErrorKind::Unsupported);
assert!(has_source::<cpueinsum::Error>(&error));
}
#[test]
fn bare_tcontract_decline_keeps_its_typed_source() {
let error = cpueinsum::tcontract::Error::Unsupported(
cpueinsum::tcontract::api::Unsupported::Reason("test decline"),
)
.into_crate_error("dot_general");
assert_eq!(error.kind(), ErrorKind::Unsupported);
assert!(has_source::<cpueinsum::tcontract::Error>(&error));
}
#[cfg(feature = "blas")]
#[test]
fn blas_workspace_is_a_typed_runtime_state_failure() {
let error =
cpueinsum_blas::BlasError::Work { need: 16, got: 8 }.into_crate_error("dot_general");
assert_eq!(error.kind(), ErrorKind::RuntimeState);
assert!(has_source::<cpueinsum_blas::BlasError>(&error));
}
#[cfg(any(feature = "native", feature = "blas"))]
mod plan_slot_reuse {
use super::super::{
BinaryCache, BinaryPlan, BufferPool, ExecutionResources, Key, Layout,
LowerContractionError, Scalar, SliceAccumulationSource,
};
use cpueinsum::tcontract::api::DType;
use std::sync::Arc;
use tenferro_tensor::DotGeneralConfig;
fn matmul_config() -> DotGeneralConfig {
DotGeneralConfig {
lhs_contracting_dims: [1].as_slice().into(),
rhs_contracting_dims: [0].as_slice().into(),
lhs_batch_dims: [].as_slice().into(),
rhs_batch_dims: [].as_slice().into(),
}
}
fn key(dtype: DType, (m, k, k_in, n): (usize, usize, usize, usize)) -> Key {
let lhs_strides = [1isize, m as isize];
let rhs_strides = [1isize, k_in as isize];
let out_strides = [1isize, m as isize];
Key {
dtype,
lhs: Layout::new(&[m, k], &lhs_strides, 0),
rhs: Layout::new(&[k_in, n], &rhs_strides, 0),
out: Layout::new(&[m, n], &out_strides, 0),
lhs_conj: false,
rhs_conj: false,
budget: 1,
}
}
fn run<T: Scalar>(
plan: &BinaryPlan<T>,
alpha: T,
beta: T,
lhs: &[T],
rhs: &[T],
out: &mut [T],
) -> crate::Result<()> {
let resources = ExecutionResources::new(None);
let mut exec = resources.exec(Default::default());
let mut buffers = BufferPool::new();
#[cfg(feature = "blas")]
let mut work = <T as crate::buffer_pool::PoolScalar>::pool_acquire_zeroed(
&mut buffers,
plan.work_len(),
);
#[cfg(feature = "native")]
let _ = &mut buffers;
plan.execute_slices_accum(
&mut exec,
alpha,
(lhs, 0),
(rhs, 0),
beta,
SliceAccumulationSource::Output,
(out, 0),
#[cfg(feature = "blas")]
&mut work,
)
.map(|_| ())
.map_err(|error| error.into_crate_error("dot_general"))
}
fn assert_matmul((m, k, n): (usize, usize, usize), (a, b, want): (&[f64], &[f64], &[f64])) {
for l in 0..n {
for i in 0..m {
let mut sum = 0.0;
for j in 0..k {
sum += a[i + m * j] * b[j + k * l];
}
assert!(
(want[i + m * l] - sum).abs() < 1e-12,
"output[{i}, {l}] = {} != {sum}",
want[i + m * l]
);
}
}
}
#[test]
fn a_shared_plan_survives_a_different_key_miss() {
let mut cache = BinaryCache::default();
let config = matmul_config();
let first_key = key(DType::F64, (2, 3, 3, 4));
let second_key = key(DType::F64, (3, 2, 2, 2));
let held = cache
.plan::<f64>(Some(0), 1, first_key.clone(), &config)
.expect("first plan");
let stored = cache
.plan::<f64>(Some(0), 1, second_key.clone(), &config)
.expect("second plan");
assert!(!Arc::ptr_eq(&held, &stored));
assert_eq!(cache.stats().misses, 2);
assert_eq!(cache.stats().hits, 0);
assert_eq!(
cache.slots[0].as_ref().map(|entry| entry.key.clone()),
Some(second_key)
);
let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0];
let b = [1.0, 0.0, -1.0, 2.0, 0.5, 1.0, 3.0, -2.0, 1.0, 1.0, 0.0, 2.0];
let mut want = [0.0f64; 8];
run(&held, 1.0, 0.0, &a, &b, &mut want).expect("the held plan still runs");
assert_matmul((2, 3, 4), (&a, &b, &want));
let a = [1.0, 2.0, -1.0, 0.5, 3.0, 4.0];
let b = [2.0, -1.0, 0.5, 1.0];
let mut want = [0.0f64; 6];
run(&stored, 1.0, 0.0, &a, &b, &mut want).expect("the new plan runs");
assert_matmul((3, 2, 2), (&a, &b, &want));
}
#[test]
fn a_different_scalar_type_replaces_the_slot_plan() {
let mut cache = BinaryCache::default();
let config = matmul_config();
let stored = cache
.plan::<f64>(Some(0), 1, key(DType::F64, (2, 2, 2, 2)), &config)
.expect("f64 plan");
drop(stored);
let stored_f32 = cache
.plan::<f32>(Some(0), 1, key(DType::F32, (2, 2, 2, 2)), &config)
.expect("f32 plan");
let a = [1.0f32, 2.0, 3.0, 4.0];
let b = [1.0f32, -1.0, 0.5, 2.0];
let mut out = [0.0f32; 4];
run(&stored_f32, 1.0, 0.0, &a, &b, &mut out).expect("the f32 plan runs");
assert_eq!(out, [-2.0, -2.0, 6.5, 9.0]);
assert_eq!(
cache.slots[0].as_ref().map(|entry| entry.key.dtype),
Some(DType::F32)
);
}
#[test]
fn a_failed_preparation_leaves_the_previous_entry_usable() {
let mut cache = BinaryCache::default();
let config = matmul_config();
let good_key = key(DType::F64, (2, 3, 3, 4));
let plan = cache
.plan::<f64>(Some(0), 1, good_key.clone(), &config)
.expect("stored plan");
let before = cache.stats();
let bad_key = key(DType::F64, (2, 3, 4, 2));
let failed = cache.plan::<f64>(Some(0), 1, bad_key, &config);
assert!(failed.is_err(), "a mismatched contraction must fail");
let after = cache.stats();
assert_eq!(after.misses, before.misses + 1);
assert_eq!(after.hits, before.hits);
assert_eq!(after.evictions, before.evictions);
assert_eq!(
cache.slots[0].as_ref().map(|entry| entry.key.clone()),
Some(good_key)
);
let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0];
let b = [
1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0,
];
let mut want = [0.0f64; 8];
run(&plan, 1.0, 0.0, &a, &b, &mut want).expect("the previous plan still runs");
assert_matmul((2, 3, 4), (&a, &b, &want));
}
}
const DEFAULT_C64_BOUND: u64 = 1 << 17;
fn every_host_class() -> [threadteam::HostClass; 6] {
use threadteam::HostClass;
match HostClass::Generic {
HostClass::X86_64Avx512
| HostClass::X86_64Avx2
| HostClass::X86_64Scalar
| HostClass::Aarch64Apple
| HostClass::Aarch64
| HostClass::Generic => [
HostClass::X86_64Avx512,
HostClass::X86_64Avx2,
HostClass::X86_64Scalar,
HostClass::Aarch64Apple,
HostClass::Aarch64,
HostClass::Generic,
],
}
}
#[test]
fn serial_native_lifts_the_c64_bound_on_every_host_class() {
let hosts = every_host_class();
for host in hosts {
let serial = default_faer_limit(host, 1);
if cfg!(feature = "native") {
assert_eq!(
serial,
cpueinsum::tcontract::FaerLimit::NONE,
"{} at a budget of one must lift the c64 bound",
host.name()
);
} else {
assert_eq!(
serial.c64,
Some(DEFAULT_C64_BOUND),
"{}: a build without `native` keeps tcontract's default",
host.name()
);
}
for budget in [2, 4] {
let limit = default_faer_limit(host, budget);
assert_eq!(
limit.c64,
Some(DEFAULT_C64_BOUND),
"{} at a budget of {budget}: only the serial budget lifts the bound",
host.name()
);
assert_eq!(limit.f32, None, "{}: f32 is unbounded", host.name());
assert_eq!(limit.f64, None, "{}: f64 is unbounded", host.name());
assert_eq!(limit.c32, None, "{}: c32 is unbounded", host.name());
}
}
}
#[cfg(feature = "native")]
mod faer_limit {
use crate::contraction::{take_last_plan_route, take_last_route_was_faer};
use crate::test_support::{cpu_engine, with_cpu_session};
use tenferro_tensor::{Complex64, DotGeneralConfig, Tensor, TensorDot, TensorRead};
use threadteam::HostClass;
fn config() -> DotGeneralConfig {
DotGeneralConfig {
lhs_contracting_dims: [1].as_slice().into(),
rhs_contracting_dims: [0].as_slice().into(),
lhs_batch_dims: [].as_slice().into(),
rhs_batch_dims: [].as_slice().into(),
}
}
fn data(len: usize, seed: usize) -> Vec<Complex64> {
(0..len)
.map(|i| {
let x = (i * 2654435761 + seed * 40503) % 1009;
Complex64::new(
x as f64 / 503.0 - 1.0,
((x * 7 + i) % 211) as f64 / 105.0 - 1.0,
)
})
.collect()
}
fn faer_f64_product(a: &[f64], b: &[f64], m: usize, k: usize, n: usize) -> Vec<f64> {
use crate::faer::{linalg::matmul::matmul, Accum, Mat, MatRef, Par};
let mut out = Mat::<f64>::zeros(m, n);
matmul(
out.as_mut(),
Accum::Replace,
MatRef::from_column_major_slice(a, m, k),
MatRef::from_column_major_slice(b, k, n),
1.0,
Par::Seq,
);
(0..n)
.flat_map(|j| (0..m).map(move |i| (i, j)))
.map(|(i, j)| *out.as_ref().get(i, j))
.collect()
}
fn faer_product(
a: &[Complex64],
b: &[Complex64],
m: usize,
k: usize,
n: usize,
) -> Vec<Complex64> {
use crate::faer::{c64, linalg::matmul::matmul, Accum, Mat, MatRef, Par};
let a: Vec<c64> = a.iter().map(|z| c64::new(z.re, z.im)).collect();
let b: Vec<c64> = b.iter().map(|z| c64::new(z.re, z.im)).collect();
let mut out = Mat::<c64>::zeros(m, n);
matmul(
out.as_mut(),
Accum::Replace,
MatRef::from_column_major_slice(&a, m, k),
MatRef::from_column_major_slice(&b, k, n),
c64::new(1.0, 0.0),
Par::Seq,
);
(0..n)
.flat_map(|j| (0..m).map(move |i| (i, j)))
.map(|(i, j)| {
let z = out.as_ref().get(i, j);
Complex64::new(z.re, z.im)
})
.collect()
}
fn ulp_distance(a: f64, b: f64) -> u64 {
if a == b {
return 0;
}
let key = |x: f64| {
let bits = x.to_bits() as i64;
if bits < 0 {
i64::MIN - bits
} else {
bits
}
};
key(a).abs_diff(key(b))
}
const GAMMA_SAFETY: f64 = 4.0;
fn run(engine: &mut crate::CpuEngine, a: &Tensor, b: &Tensor) -> Tensor {
with_cpu_session(engine, |session| {
session
.dot_general_read(
TensorRead::from_tensor(a),
TensorRead::from_tensor(b),
&config(),
)
.unwrap()
})
}
#[test]
fn serial_c64_route_takes_faer_above_the_bound() {
let host = HostClass::detect();
for (m, k, n) in [(1, 141, 282), (8, 141, 282), (141, 282, 141)] {
let (av, bv) = (data(m * k, 1), data(k * n, 2));
let a = Tensor::from_vec_col_major(vec![m, k], av.clone()).unwrap();
let b = Tensor::from_vec_col_major(vec![k, n], bv.clone()).unwrap();
let mut engine = cpu_engine(1).unwrap();
let _ = take_last_route_was_faer();
let got = run(&mut engine, &a, &b);
assert_eq!(
take_last_route_was_faer(),
Some(true),
"{m}x{k}x{n}: {} at a budget of one must route to faer",
host.name()
);
assert_eq!(
take_last_plan_route().map(|(algorithm, _)| algorithm.name()),
Some("faer"),
"{m}x{k}x{n}: the plan report must name the driver that ran"
);
let want = faer_product(&av, &bv, m, k, n);
let got = got.as_slice::<Complex64>().unwrap();
assert_eq!(got.len(), want.len());
let mut worst_steps = 0u64;
let mut max_err: f64 = 0.0;
let mut scale: f64 = 0.0;
for (g, w) in got.iter().zip(&want) {
assert!(
g.re.is_finite() && g.im.is_finite() && w.re.is_finite() && w.im.is_finite(),
"{m}x{k}x{n}: non-finite value"
);
worst_steps = worst_steps.max(ulp_distance(g.re, w.re));
worst_steps = worst_steps.max(ulp_distance(g.im, w.im));
max_err = max_err.max((g.re - w.re).abs()).max((g.im - w.im).abs());
scale = scale.max(w.re.abs()).max(w.im.abs());
}
let eps = f64::EPSILON;
let bound = GAMMA_SAFETY * k as f64 * eps * scale;
let measured = max_err / (eps * scale);
assert!(
max_err <= bound,
"{m}x{k}x{n}: max error {max_err:e} exceeds {GAMMA_SAFETY} * {k} * eps * {scale:e} = {bound:e}; measured {measured:.1} eps*scale, worst component {worst_steps} ULP"
);
println!(
"{m}x{k}x{n}: {measured:.1} eps*scale measured, {:.1} allowed, worst component {worst_steps} ULP",
GAMMA_SAFETY * k as f64
);
}
}
#[test]
fn shared_plan_cache_separates_budget_classes() {
let (m, k, n) = (8, 141, 282);
let a = Tensor::from_vec_col_major(vec![m, k], data(m * k, 1)).unwrap();
let b = Tensor::from_vec_col_major(vec![k, n], data(k * n, 2)).unwrap();
let plans = crate::CpuPlanCache::new();
let mut serial = cpu_engine(1).unwrap().with_plan_cache(plans.clone());
let mut wide = cpu_engine(4).unwrap().with_plan_cache(plans.clone());
let _ = run(&mut serial, &a, &b);
assert_eq!(take_last_route_was_faer(), Some(true));
let misses = plans.stats().misses;
let _ = run(&mut wide, &a, &b);
assert_eq!(
plans.stats().misses,
misses + 1,
"a wider budget must plan again, not reuse the serial plan"
);
assert_eq!(
take_last_route_was_faer(),
Some(false),
"a wider budget keeps tcontract's default FaerLimit"
);
let _ = run(&mut serial, &a, &b);
assert_eq!(take_last_route_was_faer(), Some(true));
}
#[test]
fn serial_f64_keeps_the_faer_route() {
for (m, k, n) in [(8, 141, 282), (141, 282, 141)] {
let real = |len: usize, seed: usize| -> Vec<f64> {
(0..len)
.map(|i| ((i * 2654435761 + seed * 40503) % 1009) as f64 / 503.0 - 1.0)
.collect()
};
let (av, bv) = (real(m * k, 1), real(k * n, 2));
let a = Tensor::from_vec_col_major(vec![m, k], av.clone()).unwrap();
let b = Tensor::from_vec_col_major(vec![k, n], bv.clone()).unwrap();
let mut engine = cpu_engine(1).unwrap();
let _ = take_last_route_was_faer();
let got = run(&mut engine, &a, &b);
assert_eq!(
take_last_route_was_faer(),
Some(true),
"{m}x{k}x{n}: f64 above the c64 bound must route to faer"
);
let want = faer_f64_product(&av, &bv, m, k, n);
let got = got.as_slice::<f64>().unwrap();
let mut max_err: f64 = 0.0;
let mut scale: f64 = 0.0;
for (g, w) in got.iter().zip(&want) {
assert!(
g.is_finite() && w.is_finite(),
"{m}x{k}x{n}: non-finite value"
);
max_err = max_err.max((g - w).abs());
scale = scale.max(w.abs());
}
let bound = GAMMA_SAFETY * k as f64 * f64::EPSILON * scale;
assert!(
max_err <= bound,
"{m}x{k}x{n}: max error {max_err:e} exceeds {bound:e}"
);
}
}
}
#[cfg(feature = "blas")]
mod blas_thin_c64 {
use crate::contraction::{plan_config, take_last_route_was_faer};
use crate::test_support::{cpu_engine, with_cpu_session};
use cpueinsum::tcontract::api::{DType, DotGeneral, LayoutSpec, OperandSpec, Problem};
use cpueinsum_blas::BlasScalar;
use tenferro_tensor::ContractionScalar;
use tenferro_tensor::{
Complex64, DotGeneralAccumulation, DotGeneralConfig, Tensor, TensorDot, TensorRead,
TensorWrite,
};
const THIN_C64_IS_ROUTED: bool = cfg!(feature = "blas-accelerate");
fn config() -> DotGeneralConfig {
DotGeneralConfig {
lhs_contracting_dims: [1].as_slice().into(),
rhs_contracting_dims: [0].as_slice().into(),
lhs_batch_dims: [].as_slice().into(),
rhs_batch_dims: [].as_slice().into(),
}
}
fn data(len: usize, seed: usize) -> Vec<Complex64> {
(0..len)
.map(|i| {
let x = (i * 2654435761 + seed * 40503) % 1009;
Complex64::new(
x as f64 / 503.0 - 1.0,
((x * 7 + i) % 211) as f64 / 105.0 - 1.0,
)
})
.collect()
}
fn run(engine: &mut crate::CpuEngine, a: &Tensor, b: &Tensor) -> Tensor {
with_cpu_session(engine, |session| {
session
.dot_general_read(
TensorRead::from_tensor(a),
TensorRead::from_tensor(b),
&config(),
)
.unwrap()
})
}
fn problem<T: BlasScalar>(dtype: DType, m: usize, k: usize, n: usize) -> Problem {
let spec = |dims: &[usize], strides: &[isize]| {
OperandSpec::new(LayoutSpec::new(dims, strides, 0).expect("valid layout"))
};
let _ = core::marker::PhantomData::<T>;
Problem::from_dot_general(
dtype,
spec(&[m, k], &[1, m as isize]),
spec(&[k, n], &[1, k as isize]),
spec(&[m, n], &[1, m as isize]),
&DotGeneral::new(&[1], &[0], &[], &[]),
)
.expect("valid product")
}
fn plans_faer<T: BlasScalar>(dtype: DType, m: usize, k: usize, n: usize) -> bool {
plans_faer_with::<T>(
dtype,
m,
k,
n,
&plan_config(threadteam::HostClass::detect(), 1),
)
}
fn plans_faer_with<T: BlasScalar>(
dtype: DType,
m: usize,
k: usize,
n: usize,
config: &cpueinsum::tcontract::PlanConfig,
) -> bool {
match crate::contraction::blas_plan::BinaryPlan::<T>::new(
&problem::<T>(dtype, m, k, n),
config,
) {
Ok(crate::contraction::blas_plan::BinaryPlan::Native(_)) => true,
Ok(crate::contraction::blas_plan::BinaryPlan::Vendor(_)) => false,
Err(error) => panic!("{m}x{k}x{n} {dtype:?}: {error}"),
}
}
#[test]
fn only_thin_c64_takes_the_faer_plan() {
for (m, k, n) in [(1, 141, 141), (1, 141, 282), (1, 1, 141), (2, 141, 282)] {
assert_eq!(
plans_faer::<Complex64>(DType::C64, m, k, n),
THIN_C64_IS_ROUTED,
"{m}x{k}x{n} c64"
);
}
for (m, k, n) in [(8, 141, 282), (256, 256, 256)] {
assert!(
!plans_faer::<Complex64>(DType::C64, m, k, n),
"{m}x{k}x{n} c64 must keep the vendor kernel"
);
}
assert!(
!plans_faer_with::<Complex64>(
DType::C64,
1,
1024,
282,
&cpueinsum::tcontract::PlanConfig::default()
),
"1x1024x282 c64 is above the c64 faer bound and must keep the vendor kernel"
);
assert!(!plans_faer::<f64>(DType::F64, 1, 141, 282));
assert!(!plans_faer::<f64>(DType::F64, 2, 141, 141));
assert!(!plans_faer::<f32>(DType::F32, 1, 141, 282));
assert!(!plans_faer::<num_complex::Complex32>(
DType::C32,
1,
141,
282
));
}
#[test]
fn thin_c64_runs_the_routed_plan_and_matches_a_scalar_product() {
for (m, k, n) in [(1, 141, 282), (2, 141, 141)] {
let (av, bv) = (data(m * k, 1), data(k * n, 2));
let a = Tensor::from_vec_col_major(vec![m, k], av.clone()).unwrap();
let b = Tensor::from_vec_col_major(vec![k, n], bv.clone()).unwrap();
let mut engine = cpu_engine(1).unwrap();
let _ = take_last_route_was_faer();
let got = run(&mut engine, &a, &b);
assert_eq!(
take_last_route_was_faer(),
Some(THIN_C64_IS_ROUTED),
"{m}x{k}x{n}: the routed plan must be the one that ran"
);
let got = got.as_slice::<Complex64>().unwrap();
let mut max_err: f64 = 0.0;
let mut scale: f64 = 0.0;
for j in 0..n {
for i in 0..m {
let mut acc = Complex64::new(0.0, 0.0);
for t in 0..k {
acc += av[i + t * m] * bv[t + j * k];
}
let g = got[i + j * m];
assert!(g.re.is_finite() && g.im.is_finite(), "non-finite value");
max_err = max_err
.max((g.re - acc.re).abs())
.max((g.im - acc.im).abs());
scale = scale.max(acc.re.abs()).max(acc.im.abs());
}
}
let bound = 4.0 * k as f64 * f64::EPSILON * scale;
assert!(
max_err <= bound,
"{m}x{k}x{n}: max error {max_err:e} exceeds {bound:e}"
);
}
}
#[test]
fn routed_plan_stays_within_the_recorded_deviation_of_the_vendor_kernel() {
use core::mem::MaybeUninit;
use cpueinsum::tcontract::SliceAccumulationSource;
use cpueinsum::Exec;
const RELATIVE_BOUND: f64 = 1e-13;
for (m, k, n) in [(1, 141, 141), (1, 141, 282), (2, 141, 141), (2, 141, 282)] {
let (av, bv) = (data(m * k, 1), data(k * n, 2));
let config = plan_config(threadteam::HostClass::detect(), 1);
let problem = problem::<Complex64>(DType::C64, m, k, n);
let native =
cpueinsum::BinaryPlan::<Complex64>::new(&problem, &config).expect("tcontract plan");
assert_eq!(
native.report().algorithm,
cpueinsum::tcontract::Algorithm::Faer,
"{m}x{k}x{n}: the baseline must be the plan the route takes"
);
let vendor = cpueinsum_blas::BinaryPlan::<Complex64>::new(&problem, &config)
.expect("vendor plan");
let mut work = vec![Complex64::new(0.0, 0.0); vendor.work_len()];
let mut fresh = vec![MaybeUninit::<Complex64>::uninit(); m * n];
let (got, _) = native
.execute_uninit_slices(
&mut Exec::serial(),
Complex64::new(1.0, 0.0),
(&av, 0),
(&bv, 0),
Complex64::new(0.0, 0.0),
SliceAccumulationSource::Absent,
(&mut fresh, 0),
)
.expect("routed execution");
let routed = got.to_vec();
let mut fresh = vec![MaybeUninit::<Complex64>::uninit(); m * n];
let (got, _) = vendor
.execute_uninit_slices(
&mut Exec::serial(),
Complex64::new(1.0, 0.0),
(&av, 0),
(&bv, 0),
Complex64::new(0.0, 0.0),
SliceAccumulationSource::Absent,
(&mut fresh, 0),
&mut work,
)
.expect("vendor execution");
let vendor_out = got.to_vec();
let scale = vendor_out
.iter()
.map(|z| z.re.abs().max(z.im.abs()))
.fold(0.0_f64, f64::max);
let max_diff = routed
.iter()
.zip(&vendor_out)
.map(|(r, v)| (r.re - v.re).abs().max((r.im - v.im).abs()))
.fold(0.0_f64, f64::max);
assert!(
max_diff <= RELATIVE_BOUND * scale,
"{m}x{k}x{n}: routed output moved {max_diff:e} from the vendor kernel, \
above {RELATIVE_BOUND:e} of its {scale:e} scale"
);
}
}
#[test]
fn thin_c64_accumulates_into_an_initialized_output() {
let (m, k, n) = (1, 141, 282);
let (av, bv) = (data(m * k, 1), data(k * n, 2));
let a = Tensor::from_vec_col_major(vec![m, k], av.clone()).unwrap();
let b = Tensor::from_vec_col_major(vec![k, n], bv.clone()).unwrap();
let prior = data(m * n, 3);
let mut out = Tensor::from_vec_col_major(vec![m, n], prior.clone()).unwrap();
let mut engine = cpu_engine(1).unwrap();
let accumulation = DotGeneralAccumulation {
lhs_conj: false,
rhs_conj: false,
alpha: ContractionScalar::C64(Complex64::new(2.0, 0.0)),
beta: ContractionScalar::C64(Complex64::new(3.0, 0.0)),
};
let _ = take_last_route_was_faer();
with_cpu_session(&mut engine, |session| {
session
.dot_general_read_into_accum(
TensorRead::from_tensor(&a),
TensorRead::from_tensor(&b),
&config(),
accumulation,
TensorWrite::from_tensor(&mut out),
)
.unwrap()
});
assert_eq!(
take_last_route_was_faer(),
Some(THIN_C64_IS_ROUTED),
"the accumulating call must run the routed plan"
);
let got = out.as_slice::<Complex64>().unwrap();
let mut max_err: f64 = 0.0;
let mut scale: f64 = 0.0;
for j in 0..n {
let mut acc = Complex64::new(0.0, 0.0);
for t in 0..k {
acc += av[t] * bv[t + j * k];
}
let want = acc * 2.0 + prior[j] * 3.0;
let g = got[j];
max_err = max_err
.max((g.re - want.re).abs())
.max((g.im - want.im).abs());
scale = scale.max(want.re.abs()).max(want.im.abs());
}
let bound = 4.0 * k as f64 * f64::EPSILON * scale * 3.0;
assert!(max_err <= bound, "max error {max_err:e} exceeds {bound:e}");
}
}