use crate::lmm::{fit_lmm, LmmFit, LmmGroupings, LmmWorkspace};
#[cfg(test)]
use crate::{ModelSpec, StartValues};
use super::common::{
assemble_varcorr, fill_se_by_predictor, nan_vcov, vcov_from_chol, FitDiagnostics,
};
#[cfg(test)]
use super::common::{to_col_major, warm_theta};
use super::{Fit, FitOptions};
#[allow(clippy::too_many_arguments)] pub(super) fn accumulate_lmm_rows(
ws: &mut LmmWorkspace,
x_mat: faer::MatRef<'_, f64>,
y: &[f64],
n: usize,
p: usize,
cluster_ids: &[u32],
extra_ids: &[Vec<u32>],
weights: Option<&[f64]>,
) {
ws.suff.reset();
ws.suff.groupings.set_slope_scales(x_mat, weights);
if n > 0 && p > 0 {
ws.suff
.add_rows_multi(x_mat, y, cluster_ids, extra_ids, weights);
}
}
pub(crate) struct LmmResultView<'a> {
fit: LmmFit,
betas: &'a [f64],
var_diag: &'a [f64],
#[allow(dead_code)]
t_sq: &'a [f64],
factor: faer::MatRef<'a, f64>,
theta: &'a [f64],
groupings: &'a LmmGroupings,
n_rows: usize,
ranef_u: &'a [f64],
kkt_grad_norm: f64,
boundary_score: Vec<f64>,
}
#[allow(dead_code)]
impl LmmResultView<'_> {
pub(crate) fn t_sq(&self) -> &[f64] {
self.t_sq
}
pub(crate) fn betas(&self) -> &[f64] {
self.betas
}
pub(crate) fn var_diag(&self) -> &[f64] {
self.var_diag
}
pub(crate) fn diagnostics(&self) -> FitDiagnostics {
FitDiagnostics {
converged: self.fit.converged,
boundary_hit: self.fit.boundary_hit,
pinned_components: self.fit.pinned_components,
pivot: self.fit.pivot,
pivot_col: self.fit.pivot_col,
ill_conditioned: self.fit.pivot < crate::lmm::PIVOT_MIN,
pirls_exhausted: 0,
final_pirls_exhausted: false,
hessian_fallback: false,
}
}
pub(crate) fn joint_t_sq(&self) -> f64 {
self.fit.joint_t_sq
}
pub(crate) fn n_eval(&self) -> usize {
self.fit.n_eval
}
pub(crate) fn dispersion(&self) -> f64 {
self.fit.sigma_sq
}
pub(crate) fn theta(&self) -> &[f64] {
self.theta
}
pub(crate) fn groupings(&self) -> &LmmGroupings {
self.groupings
}
}
pub(crate) fn lmm_run_on<'a>(
ws: &'a mut LmmWorkspace,
target_indices: &[u32],
theta_start: Option<&[f64]>,
want_score: bool,
) -> LmmResultView<'a> {
let fit = fit_lmm(ws, target_indices, theta_start);
let n_theta = ws.suff.groupings.n_theta();
let mut kkt_grad_norm = f64::NAN;
let mut boundary_score = vec![f64::NAN; n_theta];
if fit.converged {
let p = ws.suff.m - 1;
let sc = ws.suff.groupings.theta_row_scales();
if ws.dual_scratch.is_none() {
ws.dual_scratch =
crate::lmm::LmmDualScratch::for_groupings(n_theta, p, &ws.suff.groupings)
.map(Box::new);
}
{
let LmmWorkspace {
suff,
theta,
dual_scratch,
..
} = &mut *ws;
if let Some(scratch) = dual_scratch.as_deref_mut() {
let g = &suff.groupings;
let diag = g.diagonal_theta();
let mut grad = vec![0.0; n_theta];
let st = crate::lmm::reml_gradient(&theta[..n_theta], suff, scratch, &mut grad);
if matches!(st, crate::glmm::DerivStatus::Ok(_)) {
let mut acc = 0.0_f64;
for j in 0..n_theta {
let gj = grad[j];
let is_diag = diag.contains(&j);
let (lo, hi) = if is_diag {
(0.0, crate::lmm::THETA_HI)
} else {
(-crate::lmm::THETA_HI, crate::lmm::THETA_HI)
};
let pj = if theta[j] <= lo {
gj.min(0.0)
} else if theta[j] >= hi {
gj.max(0.0)
} else {
gj
};
acc = acc.max((pj * sc[j]).abs());
}
kkt_grad_norm = acc;
}
}
}
if fit.pinned_components != 0 && want_score {
if ws.hyper_scratch.is_none() {
ws.hyper_scratch =
crate::lmm::LmmHyperScratch::for_groupings(n_theta, p, &ws.suff.groupings)
.map(Box::new);
}
let LmmWorkspace {
suff,
theta,
hyper_scratch,
..
} = &mut *ws;
if let Some(scratch) = hyper_scratch.as_deref_mut() {
let g = &suff.groupings;
let diag = g.diagonal_theta();
let mut grad = vec![0.0; n_theta];
let mut hess = faer::Mat::<f64>::zeros(n_theta, n_theta);
let st = crate::lmm::reml_hessian(
&theta[..n_theta],
suff,
scratch,
&mut grad,
&mut hess,
);
if matches!(st, crate::glmm::DerivStatus::Ok(_)) {
for (kk, &ti) in diag.iter().enumerate() {
if kk < u64::BITS as usize
&& (fit.pinned_components >> kk) & 1 == 1
&& !g.diagonal_has_nonzero_below(kk, &theta[..n_theta])
{
boundary_score[ti] = 0.5 * hess[(ti, ti)] * sc[ti] * sc[ti];
}
}
}
}
}
}
LmmResultView {
ranef_u: if ws.fit.ranef_ok {
&ws.fit.ranef_u[..ws.suff.groupings.k_total]
} else {
&[]
},
fit,
betas: &ws.fit.betas,
var_diag: &ws.fit.var_diag,
t_sq: &ws.fit.t_sq,
factor: ws.fit.factor.as_ref(),
theta: &ws.theta,
groupings: &ws.suff.groupings,
n_rows: ws.suff.n_rows,
kkt_grad_norm,
boundary_score,
}
}
pub(crate) fn lmm_view_to_fit(
view: &LmmResultView<'_>,
x: &[f64],
ids: &crate::GroupIds,
n: usize,
p: usize,
opts: &FitOptions,
) -> Fit {
let lmm_fit = &view.fit;
let diag = view.diagnostics();
let beta = view.betas.to_vec();
let sigma_sq = lmm_fit.sigma_sq;
let mut se = vec![f64::NAN; p];
fill_se_by_predictor(view.var_diag, &opts.target_indices, &mut se);
let has_endpoint = lmm_fit.deviance.is_finite();
let theta_scales = view.groupings.theta_row_scales();
let tau2: Vec<f64> = if has_endpoint {
view.theta
.iter()
.zip(theta_scales.iter())
.map(|(&t, &s)| (t / s) * (t / s) * sigma_sq)
.collect()
} else {
view.theta.iter().map(|_| f64::NAN).collect()
};
let varcorr = if has_endpoint {
assemble_varcorr(view.theta, view.groupings, sigma_sq)
} else {
vec![]
};
let vcov = if has_endpoint {
vcov_from_chol(view.factor, p, &opts.target_indices, sigma_sq)
} else {
nan_vcov(p)
};
let n_rows = view.n_rows;
let n_theta = view.theta.len();
let ranef_levels = super::common::ranef_level_counts(view.groupings);
let (fitted, ranef) = if lmm_fit.converged && !view.ranef_u.is_empty() {
let ranef = super::common::assemble_ranef_sparse(view.theta, view.groupings, view.ranef_u);
let fitted = super::common::lmm_fitted(
x,
n,
p,
view.betas,
&ranef,
view.groupings,
&ids.primary,
&ids.extra,
opts.offset.as_deref(),
);
(fitted, ranef)
} else {
(vec![], vec![])
};
let mut diagnostics = super::common::materialize_diagnostics(&diag, p, &varcorr);
diagnostics.kkt_grad_norm = view.kkt_grad_norm;
let diag_scores: Vec<f64> = view
.groupings
.diagonal_theta()
.iter()
.map(|&ti| view.boundary_score[ti])
.collect();
diagnostics.boundary_score = super::common::pinned_scores(&diag_scores, &varcorr);
let mut fit = Fit {
beta,
se,
vcov,
tau2,
dispersion: sigma_sq,
diagnostics,
varcorr,
stddev_se: vec![], n_eval: lmm_fit.n_eval,
#[cfg(feature = "counters")]
counters: lmm_fit.counters,
deviance: lmm_fit.deviance,
loglik: super::common::lmm_loglik(lmm_fit.deviance, n_rows, p),
df: if has_endpoint { p + n_theta + 1 } else { 0 },
reml: true,
fitted,
ranef,
ranef_levels,
};
fit.diagnostics.singular = fit.diagnostics.singular
|| fit.has_negligible_component(&super::common::re_scale_grid(view.groupings));
if let Some(w) = &opts.weights {
fit.deviance -= w.iter().map(|v| v.ln()).sum::<f64>();
fit.loglik = super::common::lmm_loglik(fit.deviance, n, p);
}
fit
}
#[cfg(test)]
pub(super) fn fit_lmm_into(
ws: &mut LmmWorkspace,
x: &[f64],
ids: &crate::GroupIds,
n: usize,
p: usize,
opts: &FitOptions,
start: Option<&StartValues>,
) -> Fit {
let view = lmm_run_on(
ws,
&opts.target_indices,
warm_theta(start),
opts.boundary_score,
);
lmm_view_to_fit(&view, x, ids, n, p, opts)
}
#[cfg(test)]
#[allow(clippy::too_many_arguments)] pub(super) fn fit_mle(
x: &[f64],
y: &[f64],
n: usize,
p: usize,
model: &ModelSpec,
cluster_ids: &[u32],
extra_ids: &[Vec<u32>],
start: Option<&StartValues>,
opts: &FitOptions,
) -> Fit {
let re = model
.re
.as_ref()
.expect("fit_mle requires a mixed model (re: Some)");
let slope_cols: Vec<usize> = re.slopes.iter().map(|&c| c as usize).collect();
let extra_slope_cols: Vec<Vec<usize>> = re
.extra_groupings
.iter()
.map(|g| g.slopes.iter().map(|&c| c as usize).collect())
.collect();
let mut ws = LmmWorkspace::for_cluster_spec_ext(p, model, n, &slope_cols, &extra_slope_cols);
let y_shifted: Vec<f64>;
let y_eff: &[f64] = match &opts.offset {
Some(o) => {
y_shifted = y.iter().zip(o).map(|(&yi, &oi)| yi - oi).collect();
&y_shifted
}
None => y,
};
let x_mat = to_col_major(x, n, p);
accumulate_lmm_rows(
&mut ws,
x_mat.as_ref().subrows(0, n),
y_eff,
n,
p,
cluster_ids,
extra_ids,
opts.weights.as_deref(),
);
let ids = crate::GroupIds {
primary: cluster_ids.to_vec(),
extra: extra_ids.to_vec(),
};
fit_lmm_into(&mut ws, x, &ids, n, p, opts, start)
}
#[cfg(test)]
#[allow(clippy::too_many_arguments)] pub(crate) fn fit_mle_noz_pub(
x: &[f64],
y: &[f64],
n: usize,
p: usize,
sized: &ModelSpec,
cluster_ids: &[u32],
extra_ids: &[Vec<u32>],
start: Option<&StartValues>,
opts: &FitOptions,
) -> Fit {
fit_mle(x, y, n, p, sized, cluster_ids, extra_ids, start, opts)
}