use ndarray::{Array1, Array2, ArrayView1, ArrayView2, s};
use crate::faer_ndarray::FaerEigh;
use crate::inference::smooth_test::{
SmoothTestInput, SmoothTestResult, SmoothTestScale, wood_smooth_test,
};
use crate::solver::grid_spline_2d::{GridSpline2dDesign, axis_basis_at};
pub const FISSION_MAX_INTERACTION_FRACTION: f64 = 1e-4;
const INTERACTION_NUMERICAL_FLOOR: f64 = 1e-12;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum BindingNotion {
Representational,
Computational,
}
#[derive(Clone, Debug)]
pub struct AnovaBlocks {
pub mean: f64,
pub main_a: Array1<f64>,
pub main_b: Array1<f64>,
}
pub fn basis_means(phi: ArrayView2<'_, f64>) -> Array1<f64> {
let n = phi.nrows().max(1) as f64;
let mut m = Array1::<f64>::zeros(phi.ncols());
for row in phi.rows() {
for (j, &v) in row.iter().enumerate() {
m[j] += v;
}
}
m.mapv_inplace(|v| v / n);
m
}
pub fn anova_blocks(
c: ArrayView2<'_, f64>,
mean_a: ArrayView1<'_, f64>,
mean_b: ArrayView1<'_, f64>,
) -> Result<AnovaBlocks, String> {
let (m1, m2) = c.dim();
if mean_a.len() != m1 || mean_b.len() != m2 {
return Err(format!(
"anova_blocks: coefficient matrix is {m1}×{m2} but centering means have lengths {} and {}",
mean_a.len(),
mean_b.len()
));
}
let main_a = c.dot(&mean_b);
let main_b = c.t().dot(&mean_a);
let mean = mean_a.dot(&main_a);
Ok(AnovaBlocks {
mean,
main_a,
main_b,
})
}
#[derive(Clone, Debug)]
pub struct ChildDecoder {
pub constant: f64,
pub centered_coeffs: Array1<f64>,
}
impl ChildDecoder {
pub fn raw_coeffs_partition_of_unity(&self, means: ArrayView1<'_, f64>) -> Array1<f64> {
let shift = self.constant - means.dot(&self.centered_coeffs);
self.centered_coeffs.mapv(|v| v) + Array1::from_elem(self.centered_coeffs.len(), shift)
}
}
#[derive(Clone, Debug)]
pub struct FissionPlan {
pub child_a: Vec<ChildDecoder>,
pub child_b: Vec<ChildDecoder>,
pub reconstruction_defect: f64,
}
#[derive(Clone, Debug)]
pub struct CarveReport {
pub notion: BindingNotion,
pub binding_tests: Vec<Option<SmoothTestResult>>,
pub edge_p_value: Option<f64>,
pub interaction_fraction: f64,
pub fission: Option<FissionPlan>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum FissionDecision {
SplitCertifiedJoint,
SplitReconstructionOnly,
Keep,
}
const REML_LAMBDA_GRID: usize = 121;
const REML_GOLDEN_ITERS: usize = 60;
#[derive(Clone, Debug)]
pub struct TensorSurfaceFit {
pub coeffs: Vec<Array2<f64>>,
pub coeff_covariance: Vec<Array2<f64>>,
pub residual_cross_cov: Array2<f64>,
pub unit_covariance: Array2<f64>,
pub lambda: f64,
pub edf: f64,
pub residual_df: f64,
}
impl TensorSurfaceFit {
pub fn joint_covariance(&self) -> Array2<f64> {
let d_dims = self.residual_cross_cov.nrows();
let m = self.unit_covariance.nrows();
let mut joint = Array2::<f64>::zeros((d_dims * m, d_dims * m));
for d in 0..d_dims {
for e in 0..d_dims {
let s_de = self.residual_cross_cov[[d, e]];
if s_de == 0.0 {
continue;
}
for i in 0..m {
for j in 0..m {
joint[[d * m + i, e * m + j]] = s_de * self.unit_covariance[[i, j]];
}
}
}
}
joint
}
}
pub fn fit_tensor_surface(
phi_a: ArrayView2<'_, f64>,
phi_b: ArrayView2<'_, f64>,
responses: ArrayView2<'_, f64>,
) -> Result<TensorSurfaceFit, String> {
let n = phi_a.nrows();
let m1 = phi_a.ncols();
let m2 = phi_b.ncols();
let mm = m1 * m2;
let d_dims = responses.ncols();
if phi_b.nrows() != n || responses.nrows() != n {
return Err(format!(
"fit_tensor_surface: sample sizes disagree (phi_a {n}, phi_b {}, responses {})",
phi_b.nrows(),
responses.nrows()
));
}
if mm == 0 || d_dims == 0 || n < 2 {
return Err(format!(
"fit_tensor_surface: degenerate problem (n={n}, M₁M₂={mm}, D={d_dims})"
));
}
let mut x = Array2::<f64>::zeros((n, mm));
for r in 0..n {
for j in 0..m1 {
let pa = phi_a[[r, j]];
if pa == 0.0 {
continue;
}
for k in 0..m2 {
x[[r, j * m2 + k]] = pa * phi_b[[r, k]];
}
}
}
let xtx = x.t().dot(&x);
let xty = x.t().dot(&responses); let (evals, evecs) = xtx
.eigh(faer::Side::Lower)
.map_err(|e| format!("fit_tensor_surface: design eigendecomposition failed: {e:?}"))?;
let d_max = evals.iter().cloned().fold(0.0f64, f64::max);
if !(d_max > 0.0) {
return Err("fit_tensor_surface: design is identically zero".to_string());
}
let b = evecs.t().dot(&xty); let yty: Vec<f64> = (0..d_dims)
.map(|d| responses.column(d).dot(&responses.column(d)))
.collect();
let criterion = |lambda: f64| -> f64 {
let mut v = 0.0f64;
for d in 0..d_dims {
let mut prss = yty[d];
for i in 0..mm {
prss -= b[[i, d]] * b[[i, d]] / (evals[i].max(0.0) + lambda);
}
v += (n as f64) * prss.max(f64::MIN_POSITIVE).ln();
}
let mut logdet = 0.0f64;
for i in 0..mm {
logdet += (evals[i].max(0.0) + lambda).ln();
}
v + (d_dims as f64) * (logdet - (mm as f64) * lambda.ln())
};
let lo = d_max * 1e-9;
let hi = d_max * 1e9;
let mut best_idx = 0usize;
let mut best_val = f64::INFINITY;
let grid: Vec<f64> = (0..REML_LAMBDA_GRID)
.map(|i| {
let t = i as f64 / (REML_LAMBDA_GRID - 1) as f64;
lo * (hi / lo).powf(t)
})
.collect();
for (i, &lam) in grid.iter().enumerate() {
let v = criterion(lam);
if v < best_val {
best_val = v;
best_idx = i;
}
}
let mut a_log = grid[best_idx.saturating_sub(1)].ln();
let mut c_log = grid[(best_idx + 1).min(REML_LAMBDA_GRID - 1)].ln();
let phi = (5.0f64.sqrt() - 1.0) / 2.0;
let mut x1 = c_log - phi * (c_log - a_log);
let mut x2 = a_log + phi * (c_log - a_log);
let mut f1 = criterion(x1.exp());
let mut f2 = criterion(x2.exp());
for _ in 0..REML_GOLDEN_ITERS {
if f1 <= f2 {
c_log = x2;
x2 = x1;
f2 = f1;
x1 = c_log - phi * (c_log - a_log);
f1 = criterion(x1.exp());
} else {
a_log = x1;
x1 = x2;
f1 = f2;
x2 = a_log + phi * (c_log - a_log);
f2 = criterion(x2.exp());
}
}
let lambda = (0.5 * (a_log + c_log)).exp();
let mut edf = 0.0f64;
for i in 0..mm {
let d_i = evals[i].max(0.0);
edf += d_i / (d_i + lambda);
}
let residual_df = n as f64 - edf;
if residual_df < 1.0 {
return Err(format!(
"fit_tensor_surface: too few samples for the surface (n={n}, edf={edf:.2}); \
the scale estimate needs n − edf ≥ 1"
));
}
let mut beta_rot = Array2::<f64>::zeros((mm, d_dims));
for i in 0..mm {
let denom = evals[i].max(0.0) + lambda;
for d in 0..d_dims {
beta_rot[[i, d]] = b[[i, d]] / denom;
}
}
let beta = evecs.dot(&beta_rot); let fitted = x.dot(&beta); let mut residual_cross_cov = Array2::<f64>::zeros((d_dims, d_dims));
for d in 0..d_dims {
for e in d..d_dims {
let mut acc = 0.0f64;
for r in 0..n {
acc += (responses[[r, d]] - fitted[[r, d]]) * (responses[[r, e]] - fitted[[r, e]]);
}
let v = acc / residual_df;
residual_cross_cov[[d, e]] = v;
residual_cross_cov[[e, d]] = v;
}
}
let mut scaled_evecs = evecs.clone();
for i in 0..mm {
let denom = evals[i].max(0.0) + lambda;
for row in 0..mm {
scaled_evecs[[row, i]] = evecs[[row, i]] / denom;
}
}
let unit_covariance = scaled_evecs.dot(&evecs.t());
let mut coeffs = Vec::with_capacity(d_dims);
let mut coeff_covariance = Vec::with_capacity(d_dims);
for d in 0..d_dims {
let mut c = Array2::<f64>::zeros((m1, m2));
for j in 0..m1 {
for k in 0..m2 {
c[[j, k]] = beta[[j * m2 + k, d]];
}
}
coeffs.push(c);
coeff_covariance.push(&unit_covariance * residual_cross_cov[[d, d]]);
}
Ok(TensorSurfaceFit {
coeffs,
coeff_covariance,
residual_cross_cov,
unit_covariance,
lambda,
edf,
residual_df,
})
}
#[derive(Clone, Debug)]
pub struct FittedAtomCarveInput {
pub phi_a: Array2<f64>,
pub phi_b: Array2<f64>,
pub surface: TensorSurfaceFit,
pub joint_covariance: Array2<f64>,
}
impl FittedAtomCarveInput {
pub fn representational_carve_input(&self) -> CarveInput<'_> {
CarveInput {
phi_a: self.phi_a.view(),
phi_b: self.phi_b.view(),
coeffs: self.surface.coeffs.as_slice(),
coeff_covariance: Some(self.surface.coeff_covariance.as_slice()),
joint_coeff_covariance: Some(&self.joint_covariance),
kernel_a: None,
kernel_b: None,
edf: Some(self.surface.edf),
residual_df: self.surface.residual_df,
scale: SmoothTestScale::Estimated,
notion: BindingNotion::Representational,
}
}
}
pub fn carve_input_from_fitted_atom(
basis_values: ArrayView2<'_, f64>,
decoder_coefficients: ArrayView2<'_, f64>,
m_a: usize,
m_b: usize,
) -> Result<FittedAtomCarveInput, String> {
let n = basis_values.nrows();
let fused = basis_values.ncols();
let p = decoder_coefficients.ncols();
if m_a == 0 || m_b == 0 {
return Err(format!(
"carve_input_from_fitted_atom: degenerate factor sizes (m_a={m_a}, m_b={m_b})"
));
}
if m_a.checked_mul(m_b) != Some(fused) {
return Err(format!(
"carve_input_from_fitted_atom: factor sizes {m_a}×{m_b} do not multiply to the \
fused basis width {fused}"
));
}
if decoder_coefficients.nrows() != fused {
return Err(format!(
"carve_input_from_fitted_atom: decoder has {} rows but the fused basis is width {fused}",
decoder_coefficients.nrows()
));
}
if n < 2 || p == 0 {
return Err(format!(
"carve_input_from_fitted_atom: degenerate sample (n={n}, p={p})"
));
}
let mut phi_a = Array2::<f64>::zeros((n, m_a));
for j in 0..m_a {
let col = j * m_b;
for row in 0..n {
phi_a[[row, j]] = basis_values[[row, col]];
}
}
let mut phi_b = Array2::<f64>::zeros((n, m_b));
for k in 0..m_b {
for row in 0..n {
phi_b[[row, k]] = basis_values[[row, k]];
}
}
let mut max_abs = 0.0_f64;
for &v in basis_values.iter() {
max_abs = max_abs.max(v.abs());
}
let tol = 1e-9 * (1.0 + max_abs);
for j in 0..m_a {
for k in 0..m_b {
let col = j * m_b + k;
for row in 0..n {
let recon = phi_a[[row, j]] * phi_b[[row, k]];
if (recon - basis_values[[row, col]]).abs() > tol {
return Err(format!(
"carve_input_from_fitted_atom: fused basis is not the Kronecker product \
of the {m_a}×{m_b} factor split (entry [{row},{col}] = {} vs φ¹·φ² = {recon}); \
the atom is not a constant-leading product basis",
basis_values[[row, col]]
));
}
}
}
}
let reconstruction = basis_values.dot(&decoder_coefficients);
let surface = fit_tensor_surface(phi_a.view(), phi_b.view(), reconstruction.view())?;
let joint_covariance = surface.joint_covariance();
Ok(FittedAtomCarveInput {
phi_a,
phi_b,
surface,
joint_covariance,
})
}
const PAIR_COMPONENT_MAX_CELLS: usize = 32;
const PAIR_COMPONENT_MIN_CELLS: usize = 4;
fn pair_component_cells(n: usize) -> usize {
((n as f64).cbrt().ceil() as usize).clamp(PAIR_COMPONENT_MIN_CELLS, PAIR_COMPONENT_MAX_CELLS)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PairSurfaceBackend {
GridExact,
DenseRidge,
}
#[derive(Clone, Debug)]
pub struct PairSurfaceFit {
pub phi_a: Array2<f64>,
pub phi_b: Array2<f64>,
pub surface: TensorSurfaceFit,
pub backend: PairSurfaceBackend,
pub lower_corner: [f64; 2],
pub cell_widths: [f64; 2],
}
impl PairSurfaceFit {
pub fn predict(&self, dim: usize, x1: f64, x2: f64) -> Result<(f64, f64), String> {
let d_dims = self.surface.coeffs.len();
if dim >= d_dims {
return Err(format!(
"pair surface: response dimension {dim} out of range (D = {d_dims})"
));
}
if !(x1.is_finite() && x2.is_finite()) {
return Err(format!(
"pair surface: non-finite prediction point ({x1}, {x2})"
));
}
let m = self.phi_a.ncols();
let cells = m - 3;
let (j1, b1) = axis_basis_at(self.lower_corner[0], self.cell_widths[0], cells, x1);
let (j2, b2) = axis_basis_at(self.lower_corner[1], self.cell_widths[1], cells, x2);
let c = &self.surface.coeffs[dim];
let u = &self.surface.unit_covariance;
let mut mean = 0.0;
let mut quad = 0.0;
for i in 0..4 {
for j in 0..4 {
let v_ij = b1[i] * b2[j];
mean += v_ij * c[[j1 + i, j2 + j]];
let g_ij = (j1 + i) * m + (j2 + j);
for a in 0..4 {
for b in 0..4 {
quad += v_ij * b1[a] * b2[b] * u[[g_ij, (j1 + a) * m + (j2 + b)]];
}
}
}
}
Ok((mean, self.surface.residual_cross_cov[[dim, dim]] * quad))
}
}
pub fn fit_pair_surface(
x1: &[f64],
x2: &[f64],
responses: ArrayView2<'_, f64>,
) -> Result<PairSurfaceFit, String> {
let n = x1.len();
let d_dims = responses.ncols();
if x2.len() != n || responses.nrows() != n {
return Err(format!(
"fit_pair_surface: sample sizes disagree (x1 {n}, x2 {}, responses {})",
x2.len(),
responses.nrows()
));
}
if d_dims == 0 {
return Err("fit_pair_surface: no response dimensions".to_string());
}
let mut span = [0.0_f64; 2];
for (ax, xs) in [x1, x2].into_iter().enumerate() {
let mut lo = f64::INFINITY;
let mut hi = f64::NEG_INFINITY;
for &v in xs {
lo = lo.min(v);
hi = hi.max(v);
}
if !(hi > lo && hi.is_finite() && lo.is_finite()) {
return Err(format!(
"fit_pair_surface: axis {} is degenerate or non-finite ([{lo}, {hi}]); \
no pair surface exists over a collapsed axis",
ax + 1
));
}
span[ax] = hi - lo;
}
let metric = [span[0] * span[0], span[1] * span[1]];
let k = pair_component_cells(n);
let columns: Vec<Vec<f64>> = (0..d_dims).map(|d| responses.column(d).to_vec()).collect();
let column_refs: Vec<&[f64]> = columns.iter().map(Vec::as_slice).collect();
let weights = vec![1.0_f64; n];
let design = GridSpline2dDesign::build_multi(x1, x2, &column_refs, &weights, k, metric)?;
let lower_corner = design.lower_corner();
let cell_widths = design.cell_widths();
let m = design.basis_per_axis();
let mut phi_a = Array2::<f64>::zeros((n, m));
let mut phi_b = Array2::<f64>::zeros((n, m));
for r in 0..n {
let (j0, vals) = design.axis_basis(0, x1[r])?;
for (i, &v) in vals.iter().enumerate() {
phi_a[[r, j0 + i]] = v;
}
let (j0, vals) = design.axis_basis(1, x2[r])?;
for (i, &v) in vals.iter().enumerate() {
phi_b[[r, j0 + i]] = v;
}
}
match design
.fit_reml()
.and_then(|fit| design.posterior(&fit).map(|post| (fit, post)))
{
Ok((fit, post)) => {
let mm = m * m;
let mut unit_covariance = Array2::<f64>::zeros((mm, mm));
for i in 0..mm {
for j in 0..mm {
unit_covariance[[i, j]] = post.unit_covariance[i * mm + j];
}
}
let mut residual_cross_cov = Array2::<f64>::zeros((d_dims, d_dims));
for d in 0..d_dims {
for e in 0..d_dims {
residual_cross_cov[[d, e]] = post.residual_cross_cov[d * d_dims + e];
}
}
let mut coeffs = Vec::with_capacity(d_dims);
let mut coeff_covariance = Vec::with_capacity(d_dims);
for d in 0..d_dims {
let mut c = Array2::<f64>::zeros((m, m));
for j in 0..m {
for kk in 0..m {
c[[j, kk]] = fit.coeffs[d][j * m + kk];
}
}
coeffs.push(c);
coeff_covariance.push(&unit_covariance * residual_cross_cov[[d, d]]);
}
Ok(PairSurfaceFit {
phi_a,
phi_b,
surface: TensorSurfaceFit {
coeffs,
coeff_covariance,
residual_cross_cov,
unit_covariance,
lambda: fit.log_lambda.exp(),
edf: post.edf,
residual_df: post.residual_df,
},
backend: PairSurfaceBackend::GridExact,
lower_corner,
cell_widths,
})
}
Err(grid_err) => {
let surface =
fit_tensor_surface(phi_a.view(), phi_b.view(), responses).map_err(|dense_err| {
format!(
"fit_pair_surface: grid engine degenerated ({grid_err}) and the dense \
ridge fallback failed too ({dense_err})"
)
})?;
Ok(PairSurfaceFit {
phi_a,
phi_b,
surface,
backend: PairSurfaceBackend::DenseRidge,
lower_corner,
cell_widths,
})
}
}
}
pub struct CarveInput<'a> {
pub phi_a: ArrayView2<'a, f64>,
pub phi_b: ArrayView2<'a, f64>,
pub coeffs: &'a [Array2<f64>],
pub coeff_covariance: Option<&'a [Array2<f64>]>,
pub joint_coeff_covariance: Option<&'a Array2<f64>>,
pub kernel_a: Option<Array1<f64>>,
pub kernel_b: Option<Array1<f64>>,
pub edf: Option<f64>,
pub residual_df: f64,
pub scale: SmoothTestScale,
pub notion: BindingNotion,
}
pub fn carve(input: &CarveInput<'_>, alpha: f64) -> Result<CarveReport, String> {
let n = input.phi_a.nrows();
if input.phi_b.nrows() != n {
return Err(format!(
"carve: factor bases disagree on sample size ({n} vs {})",
input.phi_b.nrows()
));
}
if input.coeffs.is_empty() {
return Err("carve: no coefficient matrices supplied".to_string());
}
let m1 = input.phi_a.ncols();
let m2 = input.phi_b.ncols();
if let Some(covs) = input.coeff_covariance
&& covs.len() != input.coeffs.len()
{
return Err(format!(
"carve: {} coefficient matrices but {} covariance blocks",
input.coeffs.len(),
covs.len()
));
}
if !(alpha > 0.0 && alpha < 1.0) {
return Err(format!("carve: alpha must be in (0,1), got {alpha}"));
}
let mean_a = basis_means(input.phi_a);
let mean_b = basis_means(input.phi_b);
let phi_a_c = {
let mut p = input.phi_a.to_owned();
for mut row in p.rows_mut() {
for j in 0..m1 {
row[j] -= mean_a[j];
}
}
p
};
let phi_b_c = {
let mut p = input.phi_b.to_owned();
for mut row in p.rows_mut() {
for j in 0..m2 {
row[j] -= mean_b[j];
}
}
p
};
let proj_a = gauge_projector(m1, input.kernel_a.as_ref())?;
let proj_b = gauge_projector(m2, input.kernel_b.as_ref())?;
let gauge_kron = gauge_kron_rowmajor(&proj_a, &proj_b);
let mut child_a: Vec<ChildDecoder> = Vec::with_capacity(input.coeffs.len());
let mut child_b: Vec<ChildDecoder> = Vec::with_capacity(input.coeffs.len());
let mut binding_tests: Vec<Option<SmoothTestResult>> = Vec::with_capacity(input.coeffs.len());
let mut interaction_energy = 0.0f64;
let mut centered_energy = 0.0f64;
for (dim, c) in input.coeffs.iter().enumerate() {
if c.dim() != (m1, m2) {
return Err(format!(
"carve: coefficient matrix {dim} is {:?}, bases say ({m1}, {m2})",
c.dim()
));
}
let blocks = anova_blocks(c.view(), mean_a.view(), mean_b.view())?;
let phi_a_c_c = phi_a_c.dot(c);
let main_a_vals = phi_a_c.dot(&blocks.main_a);
let main_b_vals = phi_b_c.dot(&blocks.main_b);
for row in 0..n {
let mut f12 = 0.0f64;
for k in 0..m2 {
f12 += phi_a_c_c[[row, k]] * phi_b_c[[row, k]];
}
interaction_energy += f12 * f12;
let centered = main_a_vals[row] + main_b_vals[row] + f12;
centered_energy += centered * centered;
}
let test = match input.coeff_covariance {
None => None,
Some(covs) => binding_wald_test(
c,
&covs[dim],
&proj_a,
&proj_b,
&gauge_kron,
input.edf,
input.residual_df,
input.scale,
),
};
binding_tests.push(test);
child_a.push(ChildDecoder {
constant: blocks.mean,
centered_coeffs: blocks.main_a,
});
child_b.push(ChildDecoder {
constant: 0.0,
centered_coeffs: blocks.main_b,
});
}
let interaction_fraction = if centered_energy > 0.0 {
interaction_energy / centered_energy
} else {
0.0
};
let edge_p_value = match input.joint_coeff_covariance {
Some(joint_cov) => joint_binding_wald_test(
input.coeffs,
joint_cov,
&proj_a,
&proj_b,
&gauge_kron,
input.edf,
input.residual_df,
input.scale,
)
.map(|t| t.p_value),
None => {
let ran: Vec<f64> = binding_tests.iter().flatten().map(|t| t.p_value).collect();
ran.iter()
.cloned()
.fold(None, |acc: Option<f64>, p| {
Some(acc.map_or(p, |a| a.min(p)))
})
.map(|min_p| (min_p * ran.len() as f64).min(1.0))
}
};
let numerically_additive = interaction_fraction <= INTERACTION_NUMERICAL_FLOOR;
let binding_proven = !numerically_additive && edge_p_value.is_some_and(|p| p <= alpha);
let negligible = interaction_fraction <= FISSION_MAX_INTERACTION_FRACTION;
let fission = if negligible && !binding_proven {
Some(FissionPlan {
child_a,
child_b,
reconstruction_defect: interaction_fraction,
})
} else {
None
};
Ok(CarveReport {
notion: input.notion,
binding_tests,
edge_p_value,
interaction_fraction,
fission,
})
}
pub fn fission_decision(
representational: &CarveReport,
computational: Option<&CarveReport>,
) -> FissionDecision {
if representational.fission.is_none() {
return FissionDecision::Keep;
}
match computational {
Some(comp) => {
if comp.fission.is_some() {
FissionDecision::SplitCertifiedJoint
} else {
FissionDecision::Keep
}
}
None => FissionDecision::SplitReconstructionOnly,
}
}
fn gauge_projector(m: usize, kernel: Option<&Array1<f64>>) -> Result<Array2<f64>, String> {
let u = match kernel {
Some(k) => {
if k.len() != m {
return Err(format!(
"gauge_projector: kernel length {} != basis size {m}",
k.len()
));
}
k.clone()
}
None => Array1::<f64>::ones(m),
};
let norm_sq: f64 = u.dot(&u);
let mut p = Array2::<f64>::eye(m);
if norm_sq > 0.0 {
for i in 0..m {
for j in 0..m {
p[[i, j]] -= u[i] * u[j] / norm_sq;
}
}
}
Ok(p)
}
fn gauge_kron_rowmajor(proj_a: &Array2<f64>, proj_b: &Array2<f64>) -> Array2<f64> {
let m1 = proj_a.nrows();
let m2 = proj_b.nrows();
let mm = m1 * m2;
let mut kron = Array2::<f64>::zeros((mm, mm));
for a in 0..m1 {
for j in 0..m1 {
let pa = proj_a[[a, j]];
if pa == 0.0 {
continue;
}
for cc in 0..m2 {
for k in 0..m2 {
kron[[a * m2 + cc, j * m2 + k]] = pa * proj_b[[k, cc]];
}
}
}
}
kron
}
fn binding_wald_test(
c: &Array2<f64>,
cov: &Array2<f64>,
proj_a: &Array2<f64>,
proj_b: &Array2<f64>,
gauge_kron: &Array2<f64>,
edf: Option<f64>,
residual_df: f64,
scale: SmoothTestScale,
) -> Option<SmoothTestResult> {
let (m1, m2) = c.dim();
let mm = m1 * m2;
if cov.dim() != (mm, mm) {
return None;
}
let projected = proj_a.dot(c).dot(proj_b);
let mut z = Array1::<f64>::zeros(mm);
for j in 0..m1 {
for k in 0..m2 {
z[j * m2 + k] = projected[[j, k]];
}
}
let cov_z = gauge_kron.dot(cov).dot(&gauge_kron.t());
let quotient_rank = ((m1.saturating_sub(1)) * (m2.saturating_sub(1))).max(1) as f64;
let edf = edf.unwrap_or(quotient_rank).min(quotient_rank);
wood_smooth_test(SmoothTestInput {
beta: z.view(),
covariance: &cov_z,
influence_matrix: None,
coeff_range: 0..mm,
edf,
nullspace_dim: 0,
residual_df,
scale,
})
}
fn joint_binding_wald_test(
coeffs: &[Array2<f64>],
joint_cov: &Array2<f64>,
proj_a: &Array2<f64>,
proj_b: &Array2<f64>,
gauge_kron: &Array2<f64>,
edf: Option<f64>,
residual_df: f64,
scale: SmoothTestScale,
) -> Option<SmoothTestResult> {
let d_dims = coeffs.len();
if d_dims == 0 {
return None;
}
let (m1, m2) = coeffs[0].dim();
let mm = m1 * m2;
let total = d_dims * mm;
if joint_cov.dim() != (total, total) {
return None;
}
let mut z = Array1::<f64>::zeros(total);
for (d, c) in coeffs.iter().enumerate() {
let projected = proj_a.dot(c).dot(proj_b);
for j in 0..m1 {
for k in 0..m2 {
z[d * mm + j * m2 + k] = projected[[j, k]];
}
}
}
let mut cov_z = Array2::<f64>::zeros((total, total));
for d in 0..d_dims {
for e in 0..d_dims {
let block = joint_cov.slice(s![d * mm..(d + 1) * mm, e * mm..(e + 1) * mm]);
let transformed = gauge_kron.dot(&block).dot(&gauge_kron.t());
cov_z
.slice_mut(s![d * mm..(d + 1) * mm, e * mm..(e + 1) * mm])
.assign(&transformed);
}
}
let quotient_rank = ((m1.saturating_sub(1)) * (m2.saturating_sub(1))).max(1) as f64;
let per_dim_edf = edf.unwrap_or(quotient_rank).min(quotient_rank);
wood_smooth_test(SmoothTestInput {
beta: z.view(),
covariance: &cov_z,
influence_matrix: None,
coeff_range: 0..total,
edf: per_dim_edf * d_dims as f64,
nullspace_dim: 0,
residual_df,
scale,
})
}
#[cfg(test)]
mod tests {
use super::*;
use ndarray::array;
fn pou_basis() -> Array2<f64> {
array![
[0.7, 0.2, 0.1],
[0.2, 0.6, 0.2],
[0.1, 0.3, 0.6],
[0.5, 0.4, 0.1],
[0.1, 0.2, 0.7],
]
}
fn pou_basis_b() -> Array2<f64> {
array![
[0.6, 0.3, 0.1],
[0.1, 0.8, 0.1],
[0.3, 0.3, 0.4],
[0.2, 0.5, 0.3],
[0.4, 0.1, 0.5],
]
}
#[test]
fn anova_reparameterization_is_exact() {
let phi_a = pou_basis();
let phi_b = pou_basis_b();
let c = array![[1.3, -0.4, 0.2], [0.0, 0.8, -1.1], [2.0, 0.5, 0.3]];
let mean_a = basis_means(phi_a.view());
let mean_b = basis_means(phi_b.view());
let blocks = anova_blocks(c.view(), mean_a.view(), mean_b.view()).expect("blocks");
for row in 0..phi_a.nrows() {
let pa = phi_a.row(row);
let pb = phi_b.row(row);
let raw = pa.dot(&c.dot(&pb.to_owned()));
let pa_c: Array1<f64> = &pa.to_owned() - &mean_a;
let pb_c: Array1<f64> = &pb.to_owned() - &mean_b;
let f12 = pa_c.dot(&c.dot(&pb_c));
let rebuilt = blocks.mean + pa_c.dot(&blocks.main_a) + pb_c.dot(&blocks.main_b) + f12;
assert!(
(raw - rebuilt).abs() < 1e-12,
"row {row}: raw {raw} vs rebuilt {rebuilt}"
);
}
}
#[test]
fn planted_additive_torus_fissions_losslessly() {
let phi_a = pou_basis();
let phi_b = pou_basis_b();
let a = array![1.0, -0.5, 2.0];
let b = array![0.3, 1.7, -1.0];
let mut c = Array2::<f64>::zeros((3, 3));
for j in 0..3 {
for k in 0..3 {
c[[j, k]] = a[j] + b[k];
}
}
let input = CarveInput {
phi_a: phi_a.view(),
phi_b: phi_b.view(),
coeffs: &[c.clone()],
coeff_covariance: None,
joint_coeff_covariance: None,
kernel_a: None,
kernel_b: None,
edf: None,
residual_df: 100.0,
scale: SmoothTestScale::Known,
notion: BindingNotion::Representational,
};
let report = carve(&input, 0.05).expect("carve");
assert!(report.interaction_fraction < 1e-24);
let plan = report
.fission
.as_ref()
.expect("additive surface must fission");
assert!(plan.reconstruction_defect < 1e-24);
let mean_a = basis_means(phi_a.view());
let mean_b = basis_means(phi_b.view());
for row in 0..phi_a.nrows() {
let pa = phi_a.row(row);
let pb = phi_b.row(row);
let raw = pa.dot(&c.dot(&pb.to_owned()));
let pa_c: Array1<f64> = &pa.to_owned() - &mean_a;
let pb_c: Array1<f64> = &pb.to_owned() - &mean_b;
let child_sum = plan.child_a[0].constant
+ pa_c.dot(&plan.child_a[0].centered_coeffs)
+ plan.child_b[0].constant
+ pb_c.dot(&plan.child_b[0].centered_coeffs);
assert!((raw - child_sum).abs() < 1e-12);
}
let raw_a = plan.child_a[0].raw_coeffs_partition_of_unity(mean_a.view());
for row in 0..phi_a.nrows() {
let pa = phi_a.row(row);
let pa_c: Array1<f64> = &pa.to_owned() - &mean_a;
let via_centered =
plan.child_a[0].constant + pa_c.dot(&plan.child_a[0].centered_coeffs);
assert!((pa.dot(&raw_a) - via_centered).abs() < 1e-12);
}
}
#[test]
fn planted_bound_torus_refuses_and_test_rejects() {
let phi_a = pou_basis();
let phi_b = pou_basis_b();
let at = array![1.0, -1.0, 0.0];
let bt = array![0.0, 1.0, -1.0];
let mut c = Array2::<f64>::zeros((3, 3));
for j in 0..3 {
for k in 0..3 {
c[[j, k]] = 2.0 * at[j] * bt[k];
}
}
let cov = Array2::<f64>::eye(9) * 1e-4;
let input = CarveInput {
phi_a: phi_a.view(),
phi_b: phi_b.view(),
coeffs: &[c],
coeff_covariance: Some(std::slice::from_ref(&cov)),
joint_coeff_covariance: None,
kernel_a: None,
kernel_b: None,
edf: None,
residual_df: 100.0,
scale: SmoothTestScale::Known,
notion: BindingNotion::Representational,
};
let report = carve(&input, 0.05).expect("carve");
assert!(report.fission.is_none(), "bound surface must not fission");
assert!(report.interaction_fraction > 0.1);
let p = report.edge_p_value.expect("test ran");
assert!(p < 1e-6, "strong planted binding must reject, p = {p}");
let a = array![1.0, -0.5, 2.0];
let b = array![0.3, 1.7, -1.0];
let mut c_add = Array2::<f64>::zeros((3, 3));
for j in 0..3 {
for k in 0..3 {
c_add[[j, k]] = a[j] + b[k];
}
}
let input_add = CarveInput {
phi_a: phi_a.view(),
phi_b: phi_b.view(),
coeffs: &[c_add],
coeff_covariance: Some(std::slice::from_ref(&cov)),
joint_coeff_covariance: None,
kernel_a: None,
kernel_b: None,
edf: None,
residual_df: 100.0,
scale: SmoothTestScale::Known,
notion: BindingNotion::Representational,
};
let report_add = carve(&input_add, 0.05).expect("carve");
let p_add = report_add.edge_p_value.expect("test ran");
assert!(
p_add > 0.99,
"additive surface carries zero projected interaction, p = {p_add}"
);
assert!(report_add.fission.is_some());
}
#[test]
fn gauge_directions_do_not_enter_the_binding_test() {
let phi_a = pou_basis();
let phi_b = pou_basis_b();
let mut c = Array2::<f64>::zeros((3, 3));
let v = array![0.4, -1.2, 0.7];
let w = array![-0.9, 0.1, 0.5];
for j in 0..3 {
for k in 0..3 {
c[[j, k]] = v[k] + w[j];
}
}
let cov = Array2::<f64>::eye(9) * 1e-4;
let input = CarveInput {
phi_a: phi_a.view(),
phi_b: phi_b.view(),
coeffs: &[c],
coeff_covariance: Some(std::slice::from_ref(&cov)),
joint_coeff_covariance: None,
kernel_a: None,
kernel_b: None,
edf: None,
residual_df: 100.0,
scale: SmoothTestScale::Known,
notion: BindingNotion::Representational,
};
let report = carve(&input, 0.05).expect("carve");
assert!(report.interaction_fraction < 1e-24);
let p = report.edge_p_value.expect("test ran");
assert!(p > 0.99, "pure-gauge coefficients must not reject, p = {p}");
}
fn bernstein_pair(n: usize) -> (Array2<f64>, Array2<f64>) {
let mut phi_a = Array2::<f64>::zeros((n, 3));
let mut phi_b = Array2::<f64>::zeros((n, 3));
for t in 0..n {
let x = t as f64 / (n - 1) as f64;
let z = ((t * 17) % n) as f64 / (n - 1) as f64;
phi_a[[t, 0]] = (1.0 - x) * (1.0 - x);
phi_a[[t, 1]] = 2.0 * x * (1.0 - x);
phi_a[[t, 2]] = x * x;
phi_b[[t, 0]] = (1.0 - z) * (1.0 - z);
phi_b[[t, 1]] = 2.0 * z * (1.0 - z);
phi_b[[t, 2]] = z * z;
}
(phi_a, phi_b)
}
fn surface_values(phi_a: &Array2<f64>, phi_b: &Array2<f64>, c: &Array2<f64>) -> Array1<f64> {
let n = phi_a.nrows();
let mut y = Array1::<f64>::zeros(n);
for r in 0..n {
y[r] = phi_a.row(r).dot(&c.dot(&phi_b.row(r).to_owned()));
}
y
}
#[test]
fn tensor_surface_fit_to_carve_proves_planted_binding_jointly() {
let n = 40usize;
let (phi_a, phi_b) = bernstein_pair(n);
let at = array![1.0, -1.0, 0.0];
let bt = array![0.0, 1.0, -1.0];
let mut c0 = Array2::<f64>::zeros((3, 3));
let mut c1 = Array2::<f64>::zeros((3, 3));
let a = array![1.0, -0.5, 2.0];
let b = array![0.3, 1.7, -1.0];
for j in 0..3 {
for k in 0..3 {
c0[[j, k]] = a[j] + b[k] + 2.0 * at[j] * bt[k];
c1[[j, k]] = 0.5 * a[j] - b[k] - 1.5 * at[j] * bt[k];
}
}
let y0 = surface_values(&phi_a, &phi_b, &c0);
let y1 = surface_values(&phi_a, &phi_b, &c1);
let mut responses = Array2::<f64>::zeros((n, 2));
for t in 0..n {
responses[[t, 0]] = y0[t] + 1e-3 * (1.3 * t as f64).sin();
responses[[t, 1]] = y1[t] + 1e-3 * (2.1 * t as f64).cos();
}
let fit = fit_tensor_surface(phi_a.view(), phi_b.view(), responses.view()).expect("fit");
for j in 0..3 {
for k in 0..3 {
assert!(
(fit.coeffs[0][[j, k]] - c0[[j, k]]).abs() < 0.05,
"C₀[{j},{k}]: fit {} vs planted {}",
fit.coeffs[0][[j, k]],
c0[[j, k]]
);
}
}
let joint = fit.joint_covariance();
let mm = 9usize;
for i in 0..mm {
for j in 0..mm {
assert!((joint[[i, j]] - fit.coeff_covariance[0][[i, j]]).abs() < 1e-15);
assert!((joint[[mm + i, mm + j]] - fit.coeff_covariance[1][[i, j]]).abs() < 1e-15);
}
}
let input = CarveInput {
phi_a: phi_a.view(),
phi_b: phi_b.view(),
coeffs: &fit.coeffs,
coeff_covariance: Some(&fit.coeff_covariance),
joint_coeff_covariance: Some(&joint),
kernel_a: None,
kernel_b: None,
edf: None,
residual_df: fit.residual_df,
scale: SmoothTestScale::Estimated,
notion: BindingNotion::Representational,
};
let report = carve(&input, 0.05).expect("carve");
let p = report.edge_p_value.expect("joint test ran");
assert!(p < 1e-3, "planted joint binding must reject, p = {p}");
assert!(report.fission.is_none(), "bound surface must not fission");
assert!(report.interaction_fraction > 0.05);
}
#[test]
fn tensor_surface_fit_additive_surface_fissions() {
let n = 40usize;
let (phi_a, phi_b) = bernstein_pair(n);
let a = array![1.0, -0.5, 2.0];
let b = array![0.3, 1.7, -1.0];
let mut c_add = Array2::<f64>::zeros((3, 3));
for j in 0..3 {
for k in 0..3 {
c_add[[j, k]] = a[j] + b[k];
}
}
let y = surface_values(&phi_a, &phi_b, &c_add);
let mut responses = Array2::<f64>::zeros((n, 1));
for t in 0..n {
responses[[t, 0]] = y[t] + 1e-5 * (0.9 * t as f64).sin();
}
let fit = fit_tensor_surface(phi_a.view(), phi_b.view(), responses.view()).expect("fit");
let input = CarveInput {
phi_a: phi_a.view(),
phi_b: phi_b.view(),
coeffs: &fit.coeffs,
coeff_covariance: None,
joint_coeff_covariance: None,
kernel_a: None,
kernel_b: None,
edf: None,
residual_df: fit.residual_df,
scale: SmoothTestScale::Estimated,
notion: BindingNotion::Representational,
};
let report = carve(&input, 0.05).expect("carve");
assert!(
report.interaction_fraction < FISSION_MAX_INTERACTION_FRACTION,
"additive surface fit must carry negligible interaction \
(fraction = {})",
report.interaction_fraction
);
assert!(report.fission.is_some());
}
#[test]
fn fission_decision_distinguishes_the_quadrants() {
let splittable = CarveReport {
notion: BindingNotion::Representational,
binding_tests: vec![],
edge_p_value: None,
interaction_fraction: 0.0,
fission: Some(FissionPlan {
child_a: vec![],
child_b: vec![],
reconstruction_defect: 0.0,
}),
};
let mut comp_splittable = splittable.clone();
comp_splittable.notion = BindingNotion::Computational;
let comp_bound = CarveReport {
notion: BindingNotion::Computational,
binding_tests: vec![],
edge_p_value: Some(1e-9),
interaction_fraction: 0.4,
fission: None,
};
assert_eq!(
fission_decision(&splittable, Some(&comp_splittable)),
FissionDecision::SplitCertifiedJoint
);
assert_eq!(
fission_decision(&splittable, None),
FissionDecision::SplitReconstructionOnly
);
assert_eq!(
fission_decision(&splittable, Some(&comp_bound)),
FissionDecision::Keep
);
let kept = CarveReport {
fission: None,
..splittable.clone()
};
assert_eq!(fission_decision(&kept, None), FissionDecision::Keep);
}
fn constant_leading_factor(n: usize, m: usize, seed: u64) -> Array2<f64> {
let mut phi = Array2::<f64>::zeros((n, m));
let mut s = seed;
for row in 0..n {
phi[[row, 0]] = 1.0;
for col in 1..m {
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
let u = ((s >> 11) as f64) / ((1u64 << 53) as f64);
phi[[row, col]] = 2.0 * u - 1.0;
}
}
phi
}
#[test]
fn producer_recovers_factor_bases_and_surface_from_fused_atom() {
let n = 40;
let (m_a, m_b) = (3, 4);
let p = 2;
let phi_a = constant_leading_factor(n, m_a, 0xA993);
let phi_b = constant_leading_factor(n, m_b, 0xB993);
let mut fused = Array2::<f64>::zeros((n, m_a * m_b));
for row in 0..n {
for j in 0..m_a {
for k in 0..m_b {
fused[[row, j * m_b + k]] = phi_a[[row, j]] * phi_b[[row, k]];
}
}
}
let mut decoder = Array2::<f64>::zeros((m_a * m_b, p));
let mut s = 0xD00D_u64;
for r in 0..(m_a * m_b) {
for c in 0..p {
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
let u = ((s >> 11) as f64) / ((1u64 << 53) as f64);
decoder[[r, c]] = 2.0 * u - 1.0;
}
}
let bundle =
carve_input_from_fitted_atom(fused.view(), decoder.view(), m_a, m_b).expect("producer");
let mut max_a = 0.0_f64;
for row in 0..n {
for j in 0..m_a {
max_a = max_a.max((bundle.phi_a[[row, j]] - phi_a[[row, j]]).abs());
}
}
let mut max_b = 0.0_f64;
for row in 0..n {
for k in 0..m_b {
max_b = max_b.max((bundle.phi_b[[row, k]] - phi_b[[row, k]]).abs());
}
}
assert!(max_a < 1e-12, "phi_a recovery error {max_a:e}");
assert!(max_b < 1e-12, "phi_b recovery error {max_b:e}");
let input = bundle.representational_carve_input();
assert_eq!(input.coeffs.len(), p);
for c in input.coeffs {
assert_eq!(c.dim(), (m_a, m_b));
}
assert_eq!(bundle.joint_covariance.dim(), (p * m_a * m_b, p * m_a * m_b));
let report = carve(&input, 0.05).expect("carve on producer output");
assert_eq!(report.notion, BindingNotion::Representational);
assert!(report.edge_p_value.is_some(), "binding p-value must be produced");
}
#[test]
fn producer_rejects_non_separable_basis() {
let n = 12;
let (m_a, m_b) = (2, 2);
let mut fused = Array2::<f64>::from_elem((n, m_a * m_b), 1.0);
fused[[3, 3]] = 7.0;
let decoder = Array2::<f64>::ones((m_a * m_b, 1));
let err = carve_input_from_fitted_atom(fused.view(), decoder.view(), m_a, m_b)
.expect_err("non-separable basis must be rejected");
assert!(
err.contains("Kronecker product"),
"rejection must name the separability failure; got: {err}"
);
}
}