#![allow(
clippy::cast_precision_loss,
clippy::needless_range_loop,
clippy::similar_names,
clippy::many_single_char_names,
clippy::unreadable_literal,
clippy::cast_possible_truncation,
clippy::unnecessary_wraps
)]
use crate::cluster::{
MAX_CLUSTER_DIMENSIONS, bartlett_weight, effective_nw_lag, intern_cluster_tuples,
multiway_subset_masks, panel_hac_meat_matrix,
};
use crate::error::StatsError;
use crate::gram::{form_xtx, invert_square};
#[derive(Clone, Copy, Debug)]
pub enum SandwichKind<'a> {
Homoskedastic,
Hc0,
Hc1,
Hc2,
Hc3,
Cluster {
groups: &'a [u32],
},
Multiway {
dimensions: &'a [&'a [u32]],
},
NeweyWest {
lag: usize,
},
PanelClusterHac {
groups: &'a [u32],
time: &'a [i64],
lag: usize,
},
}
pub fn coefficient_covariance(
x_colmajor: &[f64],
nrows: usize,
ncols: usize,
residuals: &[f64],
kind: SandwichKind<'_>,
) -> Result<Vec<f64>, StatsError> {
sandwich_from_multipliers(x_colmajor, nrows, ncols, residuals, None, kind)
}
pub fn score_coefficient_covariance(
x_colmajor: &[f64],
nrows: usize,
ncols: usize,
score_multipliers: &[f64],
fisher_weights: &[f64],
kind: SandwichKind<'_>,
) -> Result<Vec<f64>, StatsError> {
if fisher_weights.len() != nrows {
return Err(StatsError::Shape { message: "fisher_weights length != nrows" });
}
sandwich_from_multipliers(
x_colmajor,
nrows,
ncols,
score_multipliers,
Some(fisher_weights),
kind,
)
}
fn sandwich_from_multipliers(
x_colmajor: &[f64],
nrows: usize,
ncols: usize,
multipliers: &[f64],
fisher_weights: Option<&[f64]>,
kind: SandwichKind<'_>,
) -> Result<Vec<f64>, StatsError> {
if multipliers.len() != nrows {
return Err(StatsError::Shape { message: "score/residual length != nrows" });
}
if x_colmajor.len() < nrows.saturating_mul(ncols) {
return Err(StatsError::Shape { message: "X buffer too short" });
}
if nrows == 0 || ncols == 0 {
return Err(StatsError::Shape { message: "covariance needs positive dimensions" });
}
let mut gram = vec![0.0; ncols * ncols];
match fisher_weights {
None => form_xtx(x_colmajor, nrows, ncols, &mut gram),
Some(w) => form_xtwx(x_colmajor, nrows, ncols, w, &mut gram),
}
let Some(bread) = invert_square(&gram, ncols) else {
return Err(StatsError::Backend("singular sandwich bread".into()));
};
match kind {
SandwichKind::Homoskedastic => {
if fisher_weights.is_some() {
return Ok(bread);
}
if nrows <= ncols {
return Err(StatsError::Shape { message: "non-positive residual df" });
}
let rss: f64 = multipliers.iter().map(|e| e * e).sum();
let sigma2 = rss / (nrows as f64 - ncols as f64);
Ok(bread.iter().map(|v| v * sigma2).collect())
}
SandwichKind::Hc0 | SandwichKind::Hc1 | SandwichKind::Hc2 | SandwichKind::Hc3 => {
let meat = hc_meat(x_colmajor, nrows, ncols, multipliers, &gram, fisher_weights, kind)?;
Ok(sandwich_product(&bread, &meat, ncols))
}
SandwichKind::Cluster { groups } => {
if groups.len() != nrows {
return Err(StatsError::Shape { message: "cluster groups length != nrows" });
}
let meat = cluster_meat(x_colmajor, nrows, ncols, multipliers, groups)?;
let g = distinct_count(groups);
let scale = cluster_finite_sample(nrows, ncols, g)?;
let meat: Vec<f64> = meat.iter().map(|v| v * scale).collect();
Ok(sandwich_product(&bread, &meat, ncols))
}
SandwichKind::Multiway { dimensions } => {
if dimensions.is_empty() {
return Err(StatsError::Shape { message: "multiway needs ≥1 dimension" });
}
for d in dimensions {
if d.len() != nrows {
return Err(StatsError::Shape {
message: "multiway dimension length != nrows",
});
}
}
let meat = multiway_meat(x_colmajor, nrows, ncols, multipliers, dimensions)?;
Ok(sandwich_product(&bread, &meat, ncols))
}
SandwichKind::NeweyWest { lag } => {
let meat = newey_west_meat(x_colmajor, nrows, ncols, multipliers, lag)?;
Ok(sandwich_product(&bread, &meat, ncols))
}
SandwichKind::PanelClusterHac { groups, time, lag } => {
if groups.len() != nrows {
return Err(StatsError::Shape { message: "panel HAC groups length != nrows" });
}
if time.len() != nrows {
return Err(StatsError::Shape { message: "panel HAC time length != nrows" });
}
if lag == 0 {
let meat = cluster_meat(x_colmajor, nrows, ncols, multipliers, groups)?;
let g = distinct_count(groups);
let scale = cluster_finite_sample(nrows, ncols, g)?;
let meat: Vec<f64> = meat.iter().map(|v| v * scale).collect();
return Ok(sandwich_product(&bread, &meat, ncols));
}
let (meat, g) =
panel_hac_meat_matrix(x_colmajor, nrows, ncols, multipliers, groups, time, lag)?;
let scale = cluster_finite_sample(nrows, ncols, g)?;
let meat: Vec<f64> = meat.iter().map(|v| v * scale).collect();
Ok(sandwich_product(&bread, &meat, ncols))
}
}
}
fn form_xtwx(x_colmajor: &[f64], nrows: usize, ncols: usize, w: &[f64], xtwx: &mut [f64]) {
xtwx[..ncols * ncols].fill(0.0);
for c1 in 0..ncols {
for c2 in c1..ncols {
let mut acc = 0.0;
let col1 = &x_colmajor[c1 * nrows..(c1 + 1) * nrows];
let col2 = &x_colmajor[c2 * nrows..(c2 + 1) * nrows];
for r in 0..nrows {
acc += w[r] * col1[r] * col2[r];
}
xtwx[c1 * ncols + c2] = acc;
if c1 != c2 {
xtwx[c2 * ncols + c1] = acc;
}
}
}
}
fn hc_meat(
x_colmajor: &[f64],
nrows: usize,
ncols: usize,
multipliers: &[f64],
gram: &[f64],
fisher_weights: Option<&[f64]>,
kind: SandwichKind<'_>,
) -> Result<Vec<f64>, StatsError> {
let hat = match kind {
SandwichKind::Hc2 | SandwichKind::Hc3 => {
Some(leverages(x_colmajor, nrows, ncols, gram, fisher_weights)?)
}
_ => None,
};
let mut meat = vec![0.0; ncols * ncols];
for i in 0..nrows {
let e = multipliers[i];
let adj = match kind {
SandwichKind::Hc0 | SandwichKind::Hc1 => e * e,
SandwichKind::Hc2 => {
let h = hat.as_ref().unwrap()[i].clamp(0.0, 1.0 - 1e-12);
(e * e) / (1.0 - h)
}
SandwichKind::Hc3 => {
let h = hat.as_ref().unwrap()[i].clamp(0.0, 1.0 - 1e-12);
let d = 1.0 - h;
(e * e) / (d * d)
}
_ => unreachable!(),
};
accumulate_xx(&mut meat, x_colmajor, nrows, ncols, i, adj);
}
if matches!(kind, SandwichKind::Hc1) {
if nrows <= ncols {
return Err(StatsError::Shape { message: "non-positive residual df" });
}
let scale = nrows as f64 / (nrows as f64 - ncols as f64);
for v in &mut meat {
*v *= scale;
}
}
Ok(meat)
}
fn leverages(
x_colmajor: &[f64],
nrows: usize,
ncols: usize,
gram: &[f64],
fisher_weights: Option<&[f64]>,
) -> Result<Vec<f64>, StatsError> {
let Some(inv) = invert_square(gram, ncols) else {
return Err(StatsError::Backend("singular gram for leverages".into()));
};
let mut h = vec![0.0; nrows];
for i in 0..nrows {
let mut tmp = vec![0.0; ncols];
for a in 0..ncols {
let mut s = 0.0;
for b in 0..ncols {
s += inv[a * ncols + b] * x_colmajor[b * nrows + i];
}
tmp[a] = s;
}
let mut hi = 0.0;
for a in 0..ncols {
hi += x_colmajor[a * nrows + i] * tmp[a];
}
if let Some(w) = fisher_weights {
hi *= w[i];
}
h[i] = hi;
}
Ok(h)
}
fn cluster_meat(
x_colmajor: &[f64],
nrows: usize,
ncols: usize,
residuals: &[f64],
groups: &[u32],
) -> Result<Vec<f64>, StatsError> {
let mut order: Vec<usize> = (0..nrows).collect();
order.sort_by_key(|&i| groups[i]);
let mut meat = vec![0.0; ncols * ncols];
let mut score = vec![0.0; ncols];
let mut idx = 0usize;
while idx < nrows {
let g = groups[order[idx]];
score.fill(0.0);
while idx < nrows && groups[order[idx]] == g {
let i = order[idx];
let e = residuals[i];
for c in 0..ncols {
score[c] += e * x_colmajor[c * nrows + i];
}
idx += 1;
}
for a in 0..ncols {
for b in 0..ncols {
meat[a * ncols + b] += score[a] * score[b];
}
}
}
Ok(meat)
}
fn cluster_finite_sample(n: usize, p: usize, g: usize) -> Result<f64, StatsError> {
if g < 2 {
return Err(StatsError::Shape {
message: "cluster-robust variance requires at least 2 clusters",
});
}
if n <= p {
return Err(StatsError::Shape { message: "non-positive residual df" });
}
Ok((g as f64 / (g as f64 - 1.0)) * ((n as f64 - 1.0) / (n as f64 - p as f64)))
}
fn multiway_meat(
x_colmajor: &[f64],
nrows: usize,
ncols: usize,
residuals: &[f64],
dimensions: &[&[u32]],
) -> Result<Vec<f64>, StatsError> {
let d = dimensions.len();
if d == 0 || d > MAX_CLUSTER_DIMENSIONS {
return Err(StatsError::Shape { message: "multiway supports 1..=4 dimensions" });
}
let mut meat = vec![0.0; ncols * ncols];
let mut abs_diag = vec![0.0; ncols];
let mut combined = vec![0u32; nrows];
for (mask, sign) in multiway_subset_masks(d) {
let g = intern_cluster_tuples(dimensions, mask, &mut combined)?;
let part = cluster_meat(x_colmajor, nrows, ncols, residuals, &combined)?;
let scale = cluster_finite_sample(nrows, ncols, g)?;
for k in 0..meat.len() {
meat[k] += sign * scale * part[k];
}
for j in 0..ncols {
abs_diag[j] += (sign * scale * part[j * ncols + j]).abs();
}
}
for j in 0..ncols {
let v = meat[j * ncols + j];
if v < 0.0 {
let tol = 64.0 * f64::EPSILON * abs_diag[j];
if (-v) <= tol {
meat[j * ncols + j] = 0.0;
} else {
return Err(StatsError::NonPositiveVariance {
message: "multiway inclusion-exclusion meat is materially negative",
});
}
}
}
Ok(meat)
}
fn newey_west_meat(
x_colmajor: &[f64],
nrows: usize,
ncols: usize,
residuals: &[f64],
lag: usize,
) -> Result<Vec<f64>, StatsError> {
let rows: Vec<usize> = (0..nrows).collect();
newey_west_meat_on_rows(x_colmajor, nrows, ncols, residuals, &rows, lag)
}
fn newey_west_meat_on_rows(
x_colmajor: &[f64],
nrows: usize,
ncols: usize,
residuals: &[f64],
rows: &[usize],
lag: usize,
) -> Result<Vec<f64>, StatsError> {
let t_len = rows.len();
let mut scores = vec![0.0; t_len * ncols];
for (t, &i) in rows.iter().enumerate() {
let e = residuals[i];
for c in 0..ncols {
scores[t * ncols + c] = e * x_colmajor[c * nrows + i];
}
}
let mut meat = vec![0.0; ncols * ncols];
for t in 0..t_len {
for a in 0..ncols {
for b in 0..ncols {
meat[a * ncols + b] += scores[t * ncols + a] * scores[t * ncols + b];
}
}
}
let l_max = effective_nw_lag(lag, t_len.saturating_sub(1));
for ell in 1..=l_max {
let w = bartlett_weight(ell, l_max);
let mut gamma = vec![0.0; ncols * ncols];
for t in ell..t_len {
for a in 0..ncols {
for b in 0..ncols {
gamma[a * ncols + b] += scores[t * ncols + a] * scores[(t - ell) * ncols + b];
}
}
}
for a in 0..ncols {
for b in 0..ncols {
let g_ab = gamma[a * ncols + b];
let g_ba = gamma[b * ncols + a];
meat[a * ncols + b] += w * (g_ab + g_ba);
}
}
}
Ok(meat)
}
fn accumulate_xx(
meat: &mut [f64],
x_colmajor: &[f64],
nrows: usize,
ncols: usize,
row: usize,
weight: f64,
) {
for a in 0..ncols {
let xa = x_colmajor[a * nrows + row];
for b in 0..ncols {
meat[a * ncols + b] += weight * xa * x_colmajor[b * nrows + row];
}
}
}
fn sandwich_product(bread: &[f64], meat: &[f64], ncols: usize) -> Vec<f64> {
let mut tmp = vec![0.0; ncols * ncols];
for i in 0..ncols {
for j in 0..ncols {
let mut s = 0.0;
for k in 0..ncols {
s += bread[i * ncols + k] * meat[k * ncols + j];
}
tmp[i * ncols + j] = s;
}
}
let mut out = vec![0.0; ncols * ncols];
for i in 0..ncols {
for j in 0..ncols {
let mut s = 0.0;
for k in 0..ncols {
s += tmp[i * ncols + k] * bread[k * ncols + j];
}
out[i * ncols + j] = s;
}
}
out
}
fn distinct_count(groups: &[u32]) -> usize {
let mut v = groups.to_vec();
v.sort_unstable();
v.dedup();
v.len()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hc0_matches_manual_two_row() {
let x = vec![1.0, 1.0, 0.0, 1.0]; let e = vec![1.0, -1.0];
let cov = coefficient_covariance(&x, 2, 2, &e, SandwichKind::Hc0).unwrap();
assert!(cov[0].is_finite() && cov[3].is_finite());
assert!(cov[0] > 0.0);
assert!(cov[3] > 0.0);
}
#[test]
fn cluster_se_exceeds_homoskedastic_under_correlation() {
let n = 80usize;
let mut x = vec![0.0; n * 2];
let mut e = vec![0.0; n];
let mut groups = vec![0u32; n];
for i in 0..n {
let g = (i / 8) as u32;
groups[i] = g;
let t = (i % 8) as f64 / 7.0;
x[i] = 1.0;
x[n + i] = t;
e[i] = f64::from(g) * 1.5 + if i % 2 == 0 { 0.05 } else { -0.05 };
}
let homo = coefficient_covariance(&x, n, 2, &e, SandwichKind::Homoskedastic).unwrap();
let cl = coefficient_covariance(&x, n, 2, &e, SandwichKind::Cluster { groups: &groups })
.unwrap();
let se_homo = homo[0].sqrt();
let se_cl = cl[0].sqrt();
assert!(se_cl > se_homo, "cluster intercept SE {se_cl} should exceed homo {se_homo}");
}
#[test]
fn newey_west_finite() {
let n = 30usize;
let mut x = vec![0.0; n * 2];
let mut e = vec![0.0; n];
for i in 0..n {
x[i] = 1.0;
x[n + i] = i as f64;
e[i] = ((i % 3) as f64) - 1.0;
}
let cov = coefficient_covariance(&x, n, 2, &e, SandwichKind::NeweyWest { lag: 2 }).unwrap();
assert!(cov.iter().all(|v| v.is_finite()));
assert!(cov[0] > 0.0);
}
#[test]
fn sandwich_kinds_match_closed_form_four_row() {
let x = vec![1.0, 1.0, 1.0, 1.0, 0.0, 1.0, 2.0, 3.0];
let e = vec![1.0, -0.5, 0.25, -0.75];
let check = |kind: SandwichKind<'_>, expected: &[f64]| {
let cov = coefficient_covariance(&x, 4, 2, &e, kind).unwrap();
for (a, b) in cov.iter().zip(expected.iter()) {
assert!((a - b).abs() < 1e-9, "got {cov:?} expected {expected:?}");
}
};
check(SandwichKind::Hc0, &[0.553125, -0.253125, -0.253125, 0.14375]);
check(SandwichKind::Hc1, &[1.10625, -0.50625, -0.50625, 0.2875]);
check(
SandwichKind::Hc2,
&[1.766369047619049, -0.8258928571428579, -0.8258928571428581, 0.47321428571428625],
);
check(
SandwichKind::Hc3,
&[5.777352607709759, -2.727465986394562, -2.7274659863945616, 1.5688775510204103],
);
check(SandwichKind::Homoskedastic, &[0.65625, -0.28125, -0.28125, 0.1875]);
check(SandwichKind::NeweyWest { lag: 1 }, &[0.411875, -0.2084375, -0.2084375, 0.124375]);
}
#[test]
fn panel_cluster_hac_exceeds_stacked_newey_west_bridge() {
let t = 40usize;
let n = 2 * t;
let mut x = vec![0.0; n * 2];
let mut e = vec![0.0; n];
let mut groups = vec![0u32; n];
for u in 0..2u32 {
let mut prev = 1.0;
for i in 0..t {
let r = (u as usize) * t + i;
groups[r] = u;
x[r] = 1.0;
x[n + r] = i as f64 / t as f64;
let innov = if i == 0 {
if u == 0 { 1.0 } else { -1.0 }
} else {
0.05 * if i % 2 == 0 { 1.0 } else { -1.0 }
};
prev = 0.9 * prev + innov;
e[r] = prev;
}
}
let homo = coefficient_covariance(&x, n, 2, &e, SandwichKind::Homoskedastic).unwrap();
let nw = coefficient_covariance(&x, n, 2, &e, SandwichKind::NeweyWest { lag: 4 }).unwrap();
let mut times = vec![0i64; n];
for u in 0..2usize {
for i in 0..t {
times[u * t + i] = i64::try_from(i).expect("panel time index fits i64");
}
}
let panel = coefficient_covariance(
&x,
n,
2,
&e,
SandwichKind::PanelClusterHac { groups: &groups, time: ×, lag: 4 },
)
.unwrap();
let se_h = homo[0].sqrt();
let se_nw = nw[0].sqrt();
let se_p = panel[0].sqrt();
assert!(se_p > se_h, "panel {se_p} vs homo {se_h}");
assert!((se_p - se_nw).abs() > 1e-6, "panel={se_p} stacked_nw={se_nw}");
}
#[test]
fn score_sandwich_unit_weights_matches_residual() {
let x = vec![1.0, 1.0, 1.0, 1.0, 0.0, 1.0, 2.0, 3.0];
let e = vec![1.0, -0.5, 0.25, -0.75];
let w = vec![1.0, 1.0, 1.0, 1.0];
let a = coefficient_covariance(&x, 4, 2, &e, SandwichKind::Hc0).unwrap();
let b = score_coefficient_covariance(&x, 4, 2, &e, &w, SandwichKind::Hc0).unwrap();
for (u, v) in a.iter().zip(b.iter()) {
assert!((u - v).abs() < 1e-12, "a={a:?} b={b:?}");
}
}
#[test]
fn multiway_packing_collision_changes_meat() {
let n = 4usize;
let x = vec![1.0; n];
let e = [1.0, -1.0, 2.0, -2.0];
let dim_a = [1u32, 0, 2, 3];
let dim_b = [0u32, 1_000_003, 4, 5];
let dims: [&[u32]; 2] = [&dim_a, &dim_b];
let cov =
coefficient_covariance(&x, n, 1, &e, SandwichKind::Multiway { dimensions: &dims })
.unwrap();
assert!(cov[0].is_finite() && cov[0] > 0.0);
let mut out = [0u32; 4];
let g = crate::cluster::intern_cluster_tuples(&dims, 0b11, &mut out).unwrap();
assert_eq!(g, 4);
let packed: Vec<u32> = dim_a
.iter()
.zip(dim_b.iter())
.map(|(&a, &b)| a.wrapping_mul(1_000_003).wrapping_add(b))
.collect();
assert_eq!(packed[0], packed[1]);
assert_ne!(out[0], out[1]);
}
#[test]
fn one_cluster_returns_error() {
let x = vec![1.0, 1.0, 1.0, 0.0, 1.0, 2.0];
let e = vec![0.5, -0.2, 0.1];
let groups = [0u32, 0, 0];
let err = coefficient_covariance(&x, 3, 2, &e, SandwichKind::Cluster { groups: &groups })
.unwrap_err();
assert!(err.to_string().contains("at least 2 clusters"), "err={err}");
}
#[test]
fn nonpositive_residual_df_errors_for_homo_and_hc1() {
let x = vec![1.0, 1.0, 0.0, 1.0];
let e = vec![1.0, -1.0];
for kind in [SandwichKind::Homoskedastic, SandwichKind::Hc1] {
let err = coefficient_covariance(&x, 2, 2, &e, kind).unwrap_err();
assert!(
err.to_string().contains("non-positive residual df"),
"kind={kind:?} err={err}"
);
}
assert!(coefficient_covariance(&x, 2, 2, &e, SandwichKind::Hc0).is_ok());
}
#[test]
fn panel_hac_one_cluster_errors() {
let n = 6usize;
let x = vec![1.0; n];
let e = vec![1.0, -1.0, 0.5, -0.5, 0.25, -0.25];
let groups = [0u32; 6];
let time = [0i64, 1, 2, 3, 4, 5];
let err = coefficient_covariance(
&x,
n,
1,
&e,
SandwichKind::PanelClusterHac { groups: &groups, time: &time, lag: 1 },
)
.unwrap_err();
assert!(err.to_string().contains("at least 2 clusters"), "err={err}");
}
#[test]
fn newey_west_oversized_lag_matches_capped_leff() {
let n = 5usize;
let x = vec![1.0; n];
let e = vec![1.0, -0.5, 0.25, -0.75, 0.1];
let capped =
coefficient_covariance(&x, n, 1, &e, SandwichKind::NeweyWest { lag: n - 1 }).unwrap();
let oversized =
coefficient_covariance(&x, n, 1, &e, SandwichKind::NeweyWest { lag: 10 }).unwrap();
assert!(
(capped[0] - oversized[0]).abs() < 1e-12,
"capped={} oversized={}",
capped[0],
oversized[0]
);
}
#[test]
fn multiway_singleton_dimension_errors() {
let n = 4usize;
let x = vec![1.0; n];
let e = vec![1.0, -1.0, 0.5, -0.5];
let dim_a = [0u32, 0, 1, 1];
let dim_b = [0u32, 0, 0, 0];
let dims: [&[u32]; 2] = [&dim_a, &dim_b];
let err =
coefficient_covariance(&x, n, 1, &e, SandwichKind::Multiway { dimensions: &dims })
.unwrap_err();
assert!(err.to_string().contains("at least 2 clusters"), "err={err}");
}
#[test]
fn panel_hac_lag_zero_matches_cluster() {
let n = 8usize;
let x = vec![1.0; n];
let e = vec![1.0, 0.5, 0.25, -1.0, -0.5, -0.25, 0.75, 0.4];
let groups = [0u32, 0, 0, 0, 1, 1, 1, 1];
let time = [0i64, 1, 2, 3, 0, 1, 2, 3];
let panel = coefficient_covariance(
&x,
n,
1,
&e,
SandwichKind::PanelClusterHac { groups: &groups, time: &time, lag: 0 },
)
.unwrap();
let cluster =
coefficient_covariance(&x, n, 1, &e, SandwichKind::Cluster { groups: &groups })
.unwrap();
assert!((panel[0] - cluster[0]).abs() < 1e-12);
}
}