use super::*;
impl BinomialLocationScaleFamily {
pub const BLOCK_T: usize = 0;
pub const BLOCK_LOG_SIGMA: usize = 1;
pub fn parameternames() -> &'static [&'static str] {
&["threshold", "log_sigma"]
}
pub fn parameter_links() -> &'static [ParameterLink] {
&[ParameterLink::InverseLink, ParameterLink::Log]
}
pub fn metadata() -> FamilyMetadata {
FamilyMetadata {
name: "binomial_location_scale",
parameternames: Self::parameternames(),
parameter_links: Self::parameter_links(),
}
}
pub(crate) fn exact_joint_supported(&self) -> bool {
self.threshold_design.is_some() && self.log_sigma_design.is_some()
}
pub(crate) fn dense_block_designs(
&self,
) -> Result<(Cow<'_, Array2<f64>>, Cow<'_, Array2<f64>>), String> {
dense_locscale_block_designs_cached(
self.threshold_design.as_ref(),
self.log_sigma_design.as_ref(),
"BinomialLocationScaleFamily",
"BinomialLocationScale",
"threshold",
&self.policy.material_policy(),
)
}
pub(crate) fn dense_block_designs_fromspecs<'a>(
&self,
specs: &'a [ParameterBlockSpec],
) -> Result<(Cow<'a, Array2<f64>>, Cow<'a, Array2<f64>>), String> {
dense_locscale_block_designs_fromspecs(
specs,
2,
"BinomialLocationScaleFamily",
"BinomialLocationScale",
Self::BLOCK_T,
Self::BLOCK_LOG_SIGMA,
"threshold",
&self.policy.material_policy(),
)
}
pub(crate) fn exact_joint_dense_block_designs<'a>(
&'a self,
specs: Option<&'a [ParameterBlockSpec]>,
) -> Result<Option<(Cow<'a, Array2<f64>>, Cow<'a, Array2<f64>>)>, String> {
if self.threshold_design.is_some() && self.log_sigma_design.is_some() {
return self.dense_block_designs().map(Some);
}
if let Some(specs) = specs {
return self.dense_block_designs_fromspecs(specs).map(Some);
}
Ok(None)
}
pub(crate) fn exact_joint_block_designs_owned(
&self,
specs: Option<&[ParameterBlockSpec]>,
) -> Result<Option<(DesignMatrix, DesignMatrix)>, String> {
let designs = if let (Some(x_t), Some(x_ls)) = (
self.threshold_design.as_ref(),
self.log_sigma_design.as_ref(),
) {
Some((x_t.clone(), x_ls.clone()))
} else if let Some(specs) = specs {
if specs.len() != 2 {
return Err(GamlssError::DimensionMismatch { reason: format!(
"BinomialLocationScaleFamily spec-aware operator path expects 2 specs, got {}",
specs.len()
) }.into());
}
Some((
specs[Self::BLOCK_T].design.clone(),
specs[Self::BLOCK_LOG_SIGMA].design.clone(),
))
} else {
None
};
let Some((x_t, x_ls)) = designs else {
return Ok(None);
};
let n = self.y.len();
if x_t.nrows() != n || x_ls.nrows() != n {
return Err(GamlssError::DimensionMismatch { reason: format!(
"BinomialLocationScaleFamily operator designs have row mismatch: y={}, threshold={}, log_sigma={}",
n,
x_t.nrows(),
x_ls.nrows()
) }.into());
}
Ok(Some((x_t, x_ls)))
}
pub(crate) fn exact_newton_joint_gradient_from_designs(
&self,
block_states: &[ParameterBlockState],
x_t: &DesignMatrix,
x_ls: &DesignMatrix,
) -> Result<ExactNewtonJointGradientEvaluation, String> {
validate_block_count::<GamlssError>("BinomialLocationScaleFamily", 2, block_states.len())?;
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
if eta_t.len() != n
|| eta_ls.len() != n
|| self.weights.len() != n
|| x_t.nrows() != n
|| x_ls.nrows() != n
{
return Err(
"BinomialLocationScaleFamily joint gradient input size mismatch".to_string(),
);
}
let core = binomial_location_scale_core(
&self.y,
&self.weights,
eta_t,
eta_ls,
None,
&self.link_kind,
)?;
let mut grad_eta_t_v = vec![0.0_f64; n];
let mut grad_eta_ls_v = vec![0.0_f64; n];
let y_slice = self.y.as_slice().expect("y must be contiguous");
let w_slice = self.weights.as_slice().expect("weights must be contiguous");
let q0_slice = core.q0.as_slice().expect("q0 must be contiguous");
let eta_t_slice = eta_t.as_slice().expect("eta_t must be contiguous");
let eta_ls_slice = eta_ls.as_slice().expect("eta_ls must be contiguous");
let link_kind = &self.link_kind;
let gradient_pairs: Result<Vec<(f64, f64)>, String> = (0..n)
.into_par_iter()
.map(|i| {
let tower = binomial_location_scale_nll_tower(
y_slice[i],
w_slice[i],
eta_t_slice[i],
eta_ls_slice[i],
q0_slice[i],
core.mu[i],
core.dmu_dq[i],
core.d2mu_dq2[i],
core.d3mu_dq3[i],
link_kind,
false,
)?;
Ok((-tower.g[0], -tower.g[1]))
})
.collect();
for (i, (g_t, g_ls)) in gradient_pairs?.into_iter().enumerate() {
grad_eta_t_v[i] = g_t;
grad_eta_ls_v[i] = g_ls;
}
let grad_eta_t = Array1::from_vec(grad_eta_t_v);
let grad_eta_ls = Array1::from_vec(grad_eta_ls_v);
let grad_t = x_t.transpose_vector_multiply(&grad_eta_t);
let grad_ls = x_ls.transpose_vector_multiply(&grad_eta_ls);
let total = grad_t.len() + grad_ls.len();
let mut gradient = Array1::<f64>::zeros(total);
gradient.slice_mut(s![0..grad_t.len()]).assign(&grad_t);
gradient.slice_mut(s![grad_t.len()..total]).assign(&grad_ls);
Ok(ExactNewtonJointGradientEvaluation {
log_likelihood: core.log_likelihood,
gradient,
})
}
pub(crate) fn exact_newton_joint_hessian_for_specs(
&self,
block_states: &[ParameterBlockState],
specs: Option<&[ParameterBlockSpec]>,
) -> Result<Option<Array2<f64>>, String> {
let Some((x_t, x_ls)) = self.exact_joint_block_designs_owned(specs)? else {
return Ok(None);
};
self.exact_newton_joint_hessian_from_design_matrices(block_states, &x_t, &x_ls)
}
pub(crate) fn exact_newton_joint_hessian_directional_derivative_for_specs(
&self,
block_states: &[ParameterBlockState],
specs: Option<&[ParameterBlockSpec]>,
d_beta_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
let Some((x_t, x_ls)) = self.exact_joint_dense_block_designs(specs)? else {
return Ok(None);
};
self.exact_newton_joint_hessian_directional_derivative_from_designs(
block_states,
&x_t,
&x_ls,
d_beta_flat,
)
}
pub(crate) fn exact_newton_joint_hessian_second_directional_derivative_for_specs(
&self,
block_states: &[ParameterBlockState],
specs: Option<&[ParameterBlockSpec]>,
d_beta_u_flat: &Array1<f64>,
d_betav_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
let Some((x_t, x_ls)) = self.exact_joint_dense_block_designs(specs)? else {
return Ok(None);
};
self.exact_newton_joint_hessiansecond_directional_derivative_from_designs(
block_states,
&x_t,
&x_ls,
d_beta_u_flat,
d_betav_flat,
)
}
pub(crate) fn expected_joint_information_from_designs(
&self,
block_states: &[ParameterBlockState],
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
) -> Result<Option<Array2<f64>>, String> {
validate_block_count::<GamlssError>("BinomialLocationScaleFamily", 2, block_states.len())?;
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
if eta_t.len() != n
|| eta_ls.len() != n
|| self.weights.len() != n
|| x_t.nrows() != n
|| x_ls.nrows() != n
{
return Err(GamlssError::DimensionMismatch {
reason: "BinomialLocationScaleFamily expected information input size mismatch"
.to_string(),
}
.into());
}
let core = binomial_location_scale_core(
&self.y,
&self.weights,
eta_t,
eta_ls,
None,
&self.link_kind,
)?;
let rows: Vec<(f64, f64, f64)> = (0..n)
.into_par_iter()
.map(|i| {
let q = nonwiggle_q_derivs(eta_t[i], core.sigma[i]);
let (f, _, _) = binomial_expected_q_information_derivatives(
self.weights[i],
core.mu[i],
core.dmu_dq[i],
core.d2mu_dq2[i],
core.d3mu_dq3[i],
);
(f * q.q_t * q.q_t, f * q.q_t * q.q_ls, f * q.q_ls * q.q_ls)
})
.collect();
let mut coeff_tt = Array1::<f64>::zeros(n);
let mut coeff_tl = Array1::<f64>::zeros(n);
let mut coeff_ll = Array1::<f64>::zeros(n);
for (i, (tt, tl, ll)) in rows.into_iter().enumerate() {
coeff_tt[i] = tt;
coeff_tl[i] = tl;
coeff_ll[i] = ll;
}
let pt = x_t.ncols();
let pls = x_ls.ncols();
let total = pt + pls;
let h_tt = xt_diag_x_dense(x_t, &coeff_tt)?;
let h_tl = xt_diag_y_dense(x_t, &coeff_tl, x_ls)?;
let h_ll = xt_diag_x_dense(x_ls, &coeff_ll)?;
let mut h = Array2::<f64>::zeros((total, total));
h.slice_mut(s![0..pt, 0..pt]).assign(&h_tt);
h.slice_mut(s![0..pt, pt..total]).assign(&h_tl);
h.slice_mut(s![pt..total, pt..total]).assign(&h_ll);
mirror_upper_to_lower(&mut h);
Ok(Some(h))
}
pub(crate) fn expected_joint_information_directional_from_designs(
&self,
block_states: &[ParameterBlockState],
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
d_beta_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
validate_block_count::<GamlssError>("BinomialLocationScaleFamily", 2, block_states.len())?;
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
if eta_t.len() != n
|| eta_ls.len() != n
|| self.weights.len() != n
|| x_t.nrows() != n
|| x_ls.nrows() != n
{
return Err(GamlssError::DimensionMismatch {
reason: "BinomialLocationScaleFamily expected dI input size mismatch".to_string(),
}
.into());
}
let pt = x_t.ncols();
let pls = x_ls.ncols();
let total = pt + pls;
if d_beta_flat.len() != total {
return Err(GamlssError::DimensionMismatch {
reason: format!(
"BinomialLocationScaleFamily expected dI direction length mismatch: got {}, expected {}",
d_beta_flat.len(),
total
),
}
.into());
}
let d_eta_t = fast_av(x_t, &d_beta_flat.slice(s![0..pt]));
let d_eta_ls = fast_av(x_ls, &d_beta_flat.slice(s![pt..total]));
let core = binomial_location_scale_core(
&self.y,
&self.weights,
eta_t,
eta_ls,
None,
&self.link_kind,
)?;
let rows: Vec<(f64, f64, f64)> = (0..n)
.into_par_iter()
.map(|i| {
let q = nonwiggle_q_derivs(eta_t[i], core.sigma[i]);
let u = nonwiggle_q_directional(q, d_eta_t[i], d_eta_ls[i]);
let (f, f1, _) = binomial_expected_q_information_derivatives(
self.weights[i],
core.mu[i],
core.dmu_dq[i],
core.d2mu_dq2[i],
core.d3mu_dq3[i],
);
let tt = f1 * u.delta_q * q.q_t * q.q_t + 2.0 * f * q.q_t * u.delta_q_t;
let tl = f1 * u.delta_q * q.q_t * q.q_ls
+ f * (u.delta_q_t * q.q_ls + q.q_t * u.delta_q_ls);
let ll = f1 * u.delta_q * q.q_ls * q.q_ls + 2.0 * f * q.q_ls * u.delta_q_ls;
(tt, tl, ll)
})
.collect();
let mut coeff_tt = Array1::<f64>::zeros(n);
let mut coeff_tl = Array1::<f64>::zeros(n);
let mut coeff_ll = Array1::<f64>::zeros(n);
for (i, (tt, tl, ll)) in rows.into_iter().enumerate() {
coeff_tt[i] = tt;
coeff_tl[i] = tl;
coeff_ll[i] = ll;
}
let d_h_tt = xt_diag_x_dense(x_t, &coeff_tt)?;
let d_h_tl = xt_diag_y_dense(x_t, &coeff_tl, x_ls)?;
let d_h_ll = xt_diag_x_dense(x_ls, &coeff_ll)?;
let mut d_h = Array2::<f64>::zeros((total, total));
d_h.slice_mut(s![0..pt, 0..pt]).assign(&d_h_tt);
d_h.slice_mut(s![0..pt, pt..total]).assign(&d_h_tl);
d_h.slice_mut(s![pt..total, pt..total]).assign(&d_h_ll);
mirror_upper_to_lower(&mut d_h);
Ok(Some(d_h))
}
pub(crate) fn expected_joint_information_second_directional_from_designs(
&self,
block_states: &[ParameterBlockState],
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
d_beta_u_flat: &Array1<f64>,
d_betav_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
validate_block_count::<GamlssError>("BinomialLocationScaleFamily", 2, block_states.len())?;
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
if eta_t.len() != n
|| eta_ls.len() != n
|| self.weights.len() != n
|| x_t.nrows() != n
|| x_ls.nrows() != n
{
return Err(GamlssError::DimensionMismatch {
reason: "BinomialLocationScaleFamily expected d2I input size mismatch".to_string(),
}
.into());
}
let pt = x_t.ncols();
let pls = x_ls.ncols();
let total = pt + pls;
if d_beta_u_flat.len() != total {
return Err(GamlssError::DimensionMismatch { reason: format!(
"BinomialLocationScaleFamily expected d2I u direction length mismatch: got {}, expected {}",
d_beta_u_flat.len(),
total
) }.into());
}
if d_betav_flat.len() != total {
return Err(GamlssError::DimensionMismatch { reason: format!(
"BinomialLocationScaleFamily expected d2I v direction length mismatch: got {}, expected {}",
d_betav_flat.len(),
total
) }.into());
}
let d_eta_t_u = fast_av(x_t, &d_beta_u_flat.slice(s![0..pt]));
let d_eta_ls_u = fast_av(x_ls, &d_beta_u_flat.slice(s![pt..total]));
let d_eta_t_v = fast_av(x_t, &d_betav_flat.slice(s![0..pt]));
let d_eta_ls_v = fast_av(x_ls, &d_betav_flat.slice(s![pt..total]));
let core = binomial_location_scale_core(
&self.y,
&self.weights,
eta_t,
eta_ls,
None,
&self.link_kind,
)?;
let rows: Vec<(f64, f64, f64)> = (0..n)
.into_par_iter()
.map(|i| {
let q = nonwiggle_q_derivs(eta_t[i], core.sigma[i]);
let (f, f1, f2) = binomial_expected_q_information_derivatives(
self.weights[i],
core.mu[i],
core.dmu_dq[i],
core.d2mu_dq2[i],
core.d3mu_dq3[i],
);
binomial_expected_location_scale_second_coefficients(
q,
f,
f1,
f2,
d_eta_t_u[i],
d_eta_ls_u[i],
d_eta_t_v[i],
d_eta_ls_v[i],
)
})
.collect();
let mut coeff_tt = Array1::<f64>::zeros(n);
let mut coeff_tl = Array1::<f64>::zeros(n);
let mut coeff_ll = Array1::<f64>::zeros(n);
for (i, (tt, tl, ll)) in rows.into_iter().enumerate() {
coeff_tt[i] = tt;
coeff_tl[i] = tl;
coeff_ll[i] = ll;
}
let d2_h_tt = xt_diag_x_dense(x_t, &coeff_tt)?;
let d2_h_tl = xt_diag_y_dense(x_t, &coeff_tl, x_ls)?;
let d2_h_ll = xt_diag_x_dense(x_ls, &coeff_ll)?;
let mut d2_h = Array2::<f64>::zeros((total, total));
d2_h.slice_mut(s![0..pt, 0..pt]).assign(&d2_h_tt);
d2_h.slice_mut(s![0..pt, pt..total]).assign(&d2_h_tl);
d2_h.slice_mut(s![pt..total, pt..total]).assign(&d2_h_ll);
mirror_upper_to_lower(&mut d2_h);
Ok(Some(d2_h))
}
pub(crate) fn expected_joint_contracted_trace_hessian_from_designs(
&self,
block_states: &[ParameterBlockState],
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
trace_weight: &Array2<f64>,
) -> Result<Option<Array2<f64>>, String> {
validate_block_count::<GamlssError>("BinomialLocationScaleFamily", 2, block_states.len())?;
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
if eta_t.len() != n
|| eta_ls.len() != n
|| self.weights.len() != n
|| x_t.nrows() != n
|| x_ls.nrows() != n
{
return Err(GamlssError::DimensionMismatch {
reason: "BinomialLocationScaleFamily expected contracted trace input size mismatch"
.to_string(),
}
.into());
}
let pt = x_t.ncols();
let pls = x_ls.ncols();
let total = pt + pls;
if trace_weight.dim() != (total, total) {
return Err(GamlssError::DimensionMismatch {
reason: format!(
"BinomialLocationScaleFamily expected contracted trace weight shape {:?} == ({total}, {total})",
trace_weight.dim()
),
}
.into());
}
let core = binomial_location_scale_core(
&self.y,
&self.weights,
eta_t,
eta_ls,
None,
&self.link_kind,
)?;
let rows: Vec<(f64, f64, f64)> = (0..n)
.into_par_iter()
.map(|i| {
let mut trace_tt = 0.0;
for a in 0..pt {
for b in 0..pt {
trace_tt += x_t[[i, a]] * trace_weight[[a, b]] * x_t[[i, b]];
}
}
let mut trace_tl = 0.0;
for a in 0..pt {
for b in 0..pls {
trace_tl += x_t[[i, a]]
* (trace_weight[[a, pt + b]] + trace_weight[[pt + b, a]])
* x_ls[[i, b]];
}
}
let mut trace_ll = 0.0;
for a in 0..pls {
for b in 0..pls {
trace_ll += x_ls[[i, a]] * trace_weight[[pt + a, pt + b]] * x_ls[[i, b]];
}
}
let q = nonwiggle_q_derivs(eta_t[i], core.sigma[i]);
let (f, f1, f2) = binomial_expected_q_information_derivatives(
self.weights[i],
core.mu[i],
core.dmu_dq[i],
core.d2mu_dq2[i],
core.d3mu_dq3[i],
);
let (tt_tt, tt_tl, tt_ll) = binomial_expected_location_scale_second_coefficients(
q, f, f1, f2, 1.0, 0.0, 1.0, 0.0,
);
let (tl_tt, tl_tl, tl_ll) = binomial_expected_location_scale_second_coefficients(
q, f, f1, f2, 1.0, 0.0, 0.0, 1.0,
);
let (ll_tt, ll_tl, ll_ll) = binomial_expected_location_scale_second_coefficients(
q, f, f1, f2, 0.0, 1.0, 0.0, 1.0,
);
(
trace_tt * tt_tt + trace_tl * tt_tl + trace_ll * tt_ll,
trace_tt * tl_tt + trace_tl * tl_tl + trace_ll * tl_ll,
trace_tt * ll_tt + trace_tl * ll_tl + trace_ll * ll_ll,
)
})
.collect();
let mut coeff_tt = Array1::<f64>::zeros(n);
let mut coeff_tl = Array1::<f64>::zeros(n);
let mut coeff_ll = Array1::<f64>::zeros(n);
for (i, (tt, tl, ll)) in rows.into_iter().enumerate() {
coeff_tt[i] = tt;
coeff_tl[i] = tl;
coeff_ll[i] = ll;
}
let h_tt = xt_diag_x_dense(x_t, &coeff_tt)?;
let h_tl = xt_diag_y_dense(x_t, &coeff_tl, x_ls)?;
let h_ll = xt_diag_x_dense(x_ls, &coeff_ll)?;
let mut h = Array2::<f64>::zeros((total, total));
h.slice_mut(s![0..pt, 0..pt]).assign(&h_tt);
h.slice_mut(s![0..pt, pt..total]).assign(&h_tl);
h.slice_mut(s![pt..total, pt..total]).assign(&h_ll);
mirror_upper_to_lower(&mut h);
Ok(Some(h))
}
pub(crate) fn expected_joint_information_for_specs(
&self,
block_states: &[ParameterBlockState],
specs: Option<&[ParameterBlockSpec]>,
) -> Result<Option<Array2<f64>>, String> {
let Some((x_t, x_ls)) = self.exact_joint_dense_block_designs(specs)? else {
return Ok(None);
};
self.expected_joint_information_from_designs(block_states, &x_t, &x_ls)
}
pub(crate) fn expected_joint_information_directional_for_specs(
&self,
block_states: &[ParameterBlockState],
specs: Option<&[ParameterBlockSpec]>,
d_beta_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
let Some((x_t, x_ls)) = self.exact_joint_dense_block_designs(specs)? else {
return Ok(None);
};
self.expected_joint_information_directional_from_designs(
block_states,
&x_t,
&x_ls,
d_beta_flat,
)
}
pub(crate) fn expected_joint_information_second_directional_for_specs(
&self,
block_states: &[ParameterBlockState],
specs: Option<&[ParameterBlockSpec]>,
d_beta_u_flat: &Array1<f64>,
d_betav_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
let Some((x_t, x_ls)) = self.exact_joint_dense_block_designs(specs)? else {
return Ok(None);
};
self.expected_joint_information_second_directional_from_designs(
block_states,
&x_t,
&x_ls,
d_beta_u_flat,
d_betav_flat,
)
}
pub(crate) fn expected_joint_contracted_trace_hessian_for_specs(
&self,
block_states: &[ParameterBlockState],
specs: Option<&[ParameterBlockSpec]>,
trace_weight: &Array2<f64>,
) -> Result<Option<Array2<f64>>, String> {
let Some((x_t, x_ls)) = self.exact_joint_dense_block_designs(specs)? else {
return Ok(None);
};
self.expected_joint_contracted_trace_hessian_from_designs(
block_states,
&x_t,
&x_ls,
trace_weight,
)
}
pub(crate) fn exact_newton_joint_psi_terms_for_specs(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
derivative_blocks: &[Vec<crate::custom_family::CustomFamilyBlockPsiDerivative>],
psi_index: usize,
) -> Result<Option<crate::custom_family::ExactNewtonJointPsiTerms>, String> {
let Some((x_t, x_ls)) = self.exact_joint_dense_block_designs(Some(specs))? else {
return Ok(None);
};
self.exact_newton_joint_psi_terms_from_designs(
block_states,
specs,
derivative_blocks,
psi_index,
&x_t,
&x_ls,
)
}
pub(crate) fn exact_newton_joint_psisecond_order_terms_for_specs(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
derivative_blocks: &[Vec<crate::custom_family::CustomFamilyBlockPsiDerivative>],
psi_i: usize,
psi_j: usize,
) -> Result<Option<crate::custom_family::ExactNewtonJointPsiSecondOrderTerms>, String> {
let Some((x_t, x_ls)) = self.exact_joint_dense_block_designs(Some(specs))? else {
return Ok(None);
};
self.exact_newton_joint_psisecond_order_terms_from_designs(
block_states,
derivative_blocks,
psi_i,
psi_j,
&x_t,
&x_ls,
)
}
pub(crate) fn exact_newton_joint_psihessian_directional_derivative_for_specs(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
derivative_blocks: &[Vec<crate::custom_family::CustomFamilyBlockPsiDerivative>],
psi_index: usize,
d_beta_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
let Some((x_t, x_ls)) = self.exact_joint_dense_block_designs(Some(specs))? else {
return Ok(None);
};
self.exact_newton_joint_psihessian_directional_derivative_from_designs(
block_states,
derivative_blocks,
psi_index,
d_beta_flat,
&x_t,
&x_ls,
)
}
pub(crate) fn exact_newton_joint_hessian_row_coefficients(
&self,
block_states: &[ParameterBlockState],
) -> Result<(Array1<f64>, Array1<f64>, Array1<f64>), String> {
validate_block_count::<GamlssError>("BinomialLocationScaleFamily", 2, block_states.len())?;
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
if eta_t.len() != n || eta_ls.len() != n || self.weights.len() != n {
return Err(GamlssError::DimensionMismatch {
reason: "BinomialLocationScaleFamily input size mismatch".to_string(),
}
.into());
}
let core = binomial_location_scale_core(
&self.y,
&self.weights,
eta_t,
eta_ls,
None,
&self.link_kind,
)?;
let mut coeff_tt = vec![0.0_f64; n];
let mut coeff_tl = vec![0.0_f64; n];
let mut coeff_ll = vec![0.0_f64; n];
let y_slice = self.y.as_slice().expect("y must be contiguous");
let w_slice = self.weights.as_slice().expect("weights must be contiguous");
let q0_slice = core.q0.as_slice().expect("q0 must be contiguous");
let sigma_slice = core.sigma.as_slice().expect("sigma must be contiguous");
let dsigma_slice = core
.dsigma_deta
.as_slice()
.expect("dsigma_deta must be contiguous");
let mu_slice = core.mu.as_slice().expect("mu must be contiguous");
let dmu_slice = core.dmu_dq.as_slice().expect("dmu_dq must be contiguous");
let d2mu_slice = core
.d2mu_dq2
.as_slice()
.expect("d2mu_dq2 must be contiguous");
let d3mu_slice = core
.d3mu_dq3
.as_slice()
.expect("d3mu_dq3 must be contiguous");
let link_kind = &self.link_kind;
coeff_tt
.par_iter_mut()
.zip(coeff_tl.par_iter_mut())
.zip(coeff_ll.par_iter_mut())
.enumerate()
.for_each(|(i, ((c_tt, c_tl), c_ll))| {
let q = q0_slice[i];
let r = 1.0 / sigma_slice[i];
let kappa = dsigma_slice[i] / sigma_slice[i];
let (m1, m2, _) = binomial_neglog_q_derivatives_dispatch(
y_slice[i],
w_slice[i],
q,
mu_slice[i],
dmu_slice[i],
d2mu_slice[i],
d3mu_slice[i],
link_kind,
);
*c_tt = m2 * r * r;
*c_tl = kappa * r * (m1 + q * m2);
*c_ll = kappa * kappa * q * (m1 + q * m2);
});
Ok((
Array1::from_vec(coeff_tt),
Array1::from_vec(coeff_tl),
Array1::from_vec(coeff_ll),
))
}
pub(crate) fn exact_newton_block_diagonal_hessians_from_design_matrices(
&self,
block_states: &[ParameterBlockState],
x_t: &DesignMatrix,
x_ls: &DesignMatrix,
) -> Result<(Array2<f64>, Array2<f64>), String> {
let (coeff_tt, _coeff_tl, coeff_ll) =
self.exact_newton_joint_hessian_row_coefficients(block_states)?;
let h_tt = xt_diag_x_design(x_t, &coeff_tt)?;
let h_ll = xt_diag_x_design(x_ls, &coeff_ll)?;
Ok((h_tt, h_ll))
}
pub(crate) fn exact_newton_joint_hessian_from_designs(
&self,
block_states: &[ParameterBlockState],
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
) -> Result<Option<Array2<f64>>, String> {
let (coeff_tt, coeff_tl, coeff_ll) =
self.exact_newton_joint_hessian_row_coefficients(block_states)?;
let pt = x_t.ncols();
let pls = x_ls.ncols();
let h_tt = xt_diag_x_dense(x_t, &coeff_tt)?;
let h_tl = xt_diag_y_dense(x_t, &coeff_tl, x_ls)?;
let h_ll = xt_diag_x_dense(x_ls, &coeff_ll)?;
let total = pt + pls;
let mut h = Array2::<f64>::zeros((total, total));
h.slice_mut(s![0..pt, 0..pt]).assign(&h_tt);
h.slice_mut(s![0..pt, pt..total]).assign(&h_tl);
h.slice_mut(s![pt..total, pt..total]).assign(&h_ll);
mirror_upper_to_lower(&mut h);
Ok(Some(h))
}
pub(crate) fn exact_newton_joint_hessian_from_design_matrices(
&self,
block_states: &[ParameterBlockState],
x_t: &DesignMatrix,
x_ls: &DesignMatrix,
) -> Result<Option<Array2<f64>>, String> {
if let (Some(x_t_dense), Some(x_ls_dense)) = (x_t.as_dense_ref(), x_ls.as_dense_ref()) {
return self.exact_newton_joint_hessian_from_designs(
block_states,
x_t_dense,
x_ls_dense,
);
}
let (coeff_tt, coeff_tl, coeff_ll) =
self.exact_newton_joint_hessian_row_coefficients(block_states)?;
let pt = x_t.ncols();
let pls = x_ls.ncols();
let h_tt = xt_diag_x_design(x_t, &coeff_tt)?;
let h_tl = xt_diag_y_design(x_t, &coeff_tl, x_ls)?;
let h_ll = xt_diag_x_design(x_ls, &coeff_ll)?;
let total = pt + pls;
let mut h = Array2::<f64>::zeros((total, total));
h.slice_mut(s![0..pt, 0..pt]).assign(&h_tt);
h.slice_mut(s![0..pt, pt..total]).assign(&h_tl);
h.slice_mut(s![pt..total, pt..total]).assign(&h_ll);
mirror_upper_to_lower(&mut h);
Ok(Some(h))
}
pub(crate) fn exact_newton_joint_hessian_directional_derivative_from_designs(
&self,
block_states: &[ParameterBlockState],
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
d_beta_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
validate_block_count::<GamlssError>("BinomialLocationScaleFamily", 2, block_states.len())?;
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
if eta_t.len() != n || eta_ls.len() != n || self.weights.len() != n {
return Err(GamlssError::DimensionMismatch {
reason: "BinomialLocationScaleFamily input size mismatch".to_string(),
}
.into());
}
let pt = x_t.ncols();
let pls = x_ls.ncols();
if d_beta_flat.len() != pt + pls {
return Err(GamlssError::DimensionMismatch {
reason: format!(
"BinomialLocationScaleFamily joint d_beta length mismatch: got {}, expected {}",
d_beta_flat.len(),
pt + pls
),
}
.into());
}
let d_eta_t = fast_av(x_t, &d_beta_flat.slice(s![0..pt]));
let d_eta_ls = fast_av(x_ls, &d_beta_flat.slice(s![pt..pt + pls]));
let core = binomial_location_scale_core(
&self.y,
&self.weights,
eta_t,
eta_ls,
None,
&self.link_kind,
)?;
let (coeff_tt, coeff_tl, coeff_ll) =
binomial_location_scale_first_directional_coefficients(
&self.y,
&self.weights,
&core,
&d_eta_t,
&d_eta_ls,
&self.link_kind,
)?;
let d_h_tt = xt_diag_x_dense(x_t, &coeff_tt)?;
let d_h_tl = xt_diag_y_dense(x_t, &coeff_tl, x_ls)?;
let d_h_ll = xt_diag_x_dense(x_ls, &coeff_ll)?;
let total = pt + pls;
let mut d_h = Array2::<f64>::zeros((total, total));
d_h.slice_mut(s![0..pt, 0..pt]).assign(&d_h_tt);
d_h.slice_mut(s![0..pt, pt..total]).assign(&d_h_tl);
d_h.slice_mut(s![pt..total, pt..total]).assign(&d_h_ll);
mirror_upper_to_lower(&mut d_h);
Ok(Some(d_h))
}
pub(crate) fn exact_newton_joint_hessiansecond_directional_derivative_from_designs(
&self,
block_states: &[ParameterBlockState],
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
d_beta_u_flat: &Array1<f64>,
d_betav_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
validate_block_count::<GamlssError>("BinomialLocationScaleFamily", 2, block_states.len())?;
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
if eta_t.len() != n || eta_ls.len() != n || self.weights.len() != n {
return Err(GamlssError::DimensionMismatch {
reason: "BinomialLocationScaleFamily input size mismatch".to_string(),
}
.into());
}
let pt = x_t.ncols();
let pls = x_ls.ncols();
let total = pt + pls;
if d_beta_u_flat.len() != total {
return Err(GamlssError::DimensionMismatch { reason: format!(
"BinomialLocationScaleFamily joint d_beta_u length mismatch: got {}, expected {}",
d_beta_u_flat.len(),
total
) }.into());
}
if d_betav_flat.len() != total {
return Err(GamlssError::DimensionMismatch { reason: format!(
"BinomialLocationScaleFamily joint d_betav length mismatch: got {}, expected {}",
d_betav_flat.len(),
total
) }.into());
}
let d_eta_t_u = fast_av(x_t, &d_beta_u_flat.slice(s![0..pt]));
let d_eta_ls_u = fast_av(x_ls, &d_beta_u_flat.slice(s![pt..total]));
let d_eta_tv = fast_av(x_t, &d_betav_flat.slice(s![0..pt]));
let d_eta_lsv = fast_av(x_ls, &d_betav_flat.slice(s![pt..total]));
let core = binomial_location_scale_core(
&self.y,
&self.weights,
eta_t,
eta_ls,
None,
&self.link_kind,
)?;
let (coeff_tt, coeff_tl, coeff_ll) =
binomial_location_scalesecond_directional_coefficients(
&self.y,
&self.weights,
&core,
&d_eta_t_u,
&d_eta_ls_u,
&d_eta_tv,
&d_eta_lsv,
&self.link_kind,
)?;
let d2_h_tt = xt_diag_x_dense(x_t, &coeff_tt)?;
let d2_h_tl = xt_diag_y_dense(x_t, &coeff_tl, x_ls)?;
let d2_h_ll = xt_diag_x_dense(x_ls, &coeff_ll)?;
let mut d2_h = Array2::<f64>::zeros((total, total));
d2_h.slice_mut(s![0..pt, 0..pt]).assign(&d2_h_tt);
d2_h.slice_mut(s![0..pt, pt..total]).assign(&d2_h_tl);
d2_h.slice_mut(s![pt..total, pt..total]).assign(&d2_h_ll);
mirror_upper_to_lower(&mut d2_h);
Ok(Some(d2_h))
}
pub(crate) fn exact_newton_joint_psi_direction(
&self,
block_states: &[ParameterBlockState],
derivative_blocks: &[Vec<crate::custom_family::CustomFamilyBlockPsiDerivative>],
psi_index: usize,
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
policy: &crate::resource::ResourcePolicy,
) -> Result<Option<LocationScaleJointPsiDirection>, String> {
let Some(parts) = locscale_joint_psi_direction_parts(
block_states,
derivative_blocks,
psi_index,
self.y.len(),
x_t.ncols(),
x_ls.ncols(),
Self::BLOCK_T,
Self::BLOCK_LOG_SIGMA,
2,
"BinomialLocationScaleFamily",
"threshold",
policy,
)?
else {
return Ok(None);
};
Ok(Some(LocationScaleJointPsiDirection {
block_idx: parts.block_idx,
local_idx: parts.local_idx,
x_primary_psi: parts.primary_psi,
x_ls_psi: parts.log_sigma_psi,
z_primary_psi: parts.primary_z,
z_ls_psi: parts.log_sigma_z,
}))
}
pub(crate) fn exact_newton_joint_psisecond_design_drifts(
&self,
block_states: &[ParameterBlockState],
derivative_blocks: &[Vec<crate::custom_family::CustomFamilyBlockPsiDerivative>],
psi_a: &LocationScaleJointPsiDirection,
psi_b: &LocationScaleJointPsiDirection,
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
) -> Result<LocationScaleJointPsiSecondDrifts, String> {
locscale_joint_psisecond_design_drifts(
block_states,
derivative_blocks,
psi_a,
psi_b,
LocScalePsiDriftConfig {
n: self.y.len(),
p_primary: x_t.ncols(),
p_log_sigma: x_ls.ncols(),
primary_block_idx: Self::BLOCK_T,
log_sigma_block_idx: Self::BLOCK_LOG_SIGMA,
family_name: "BinomialLocationScaleFamily",
primary_label: "threshold",
policy: &self.policy,
},
)
}
pub(crate) fn exact_newton_joint_psi_terms_from_designs(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
derivative_blocks: &[Vec<crate::custom_family::CustomFamilyBlockPsiDerivative>],
psi_index: usize,
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
) -> Result<Option<crate::custom_family::ExactNewtonJointPsiTerms>, String> {
validate_block_count::<GamlssError>("BinomialLocationScaleFamily", 2, block_states.len())?;
if specs.len() != 2 || derivative_blocks.len() != 2 {
return Err(GamlssError::DimensionMismatch { reason: format!(
"BinomialLocationScaleFamily joint psi terms expect 2 specs and 2 derivative blocks, got {} and {}",
specs.len(),
derivative_blocks.len()
) }.into());
}
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
if eta_t.len() != n || eta_ls.len() != n || self.weights.len() != n {
return Err(GamlssError::DimensionMismatch {
reason: "BinomialLocationScaleFamily input size mismatch".to_string(),
}
.into());
}
let core = binomial_location_scale_core(
&self.y,
&self.weights,
eta_t,
eta_ls,
None,
&self.link_kind,
)?;
let pt = x_t.ncols();
let pls = x_ls.ncols();
let total = pt + pls;
let Some(dir_a) = self.exact_newton_joint_psi_direction(
block_states,
derivative_blocks,
psi_index,
x_t,
x_ls,
&self.policy,
)?
else {
return Ok(None);
};
let (z_t, z_ls) = (&dir_a.z_primary_psi, &dir_a.z_ls_psi);
struct PsiTermsRow {
pub(crate) r_t: f64,
pub(crate) r_ls: f64,
pub(crate) dr_t: f64,
pub(crate) dr_ls: f64,
pub(crate) h_tt: f64,
pub(crate) h_tl: f64,
pub(crate) h_ll: f64,
pub(crate) dh_tt: f64,
pub(crate) dh_tl: f64,
pub(crate) dh_ll: f64,
pub(crate) obj: f64,
}
let y_p = self.y.as_slice().expect("y must be contiguous");
let w_p = self.weights.as_slice().expect("weights must be contiguous");
let q0_p = core.q0.as_slice().expect("q0 must be contiguous");
let sigma_p = core.sigma.as_slice().expect("sigma must be contiguous");
let dsigma_p = core
.dsigma_deta
.as_slice()
.expect("dsigma_deta must be contiguous");
let mu_p = core.mu.as_slice().expect("mu must be contiguous");
let dmu_p = core.dmu_dq.as_slice().expect("dmu_dq must be contiguous");
let d2mu_p = core
.d2mu_dq2
.as_slice()
.expect("d2mu_dq2 must be contiguous");
let d3mu_p = core
.d3mu_dq3
.as_slice()
.expect("d3mu_dq3 must be contiguous");
let z_t_p = z_t.as_slice().expect("z_t must be contiguous");
let z_ls_p = z_ls.as_slice().expect("z_ls must be contiguous");
let link_kind_p = &self.link_kind;
let rows: Vec<PsiTermsRow> = (0..n)
.into_par_iter()
.map(|i| {
let q = q0_p[i];
let r = 1.0 / sigma_p[i];
let s = dsigma_p[i] / sigma_p[i];
let sz = s * z_ls_p[i];
let q_psi = -r * z_t_p[i] - q * sz;
let (a, b, c) = binomial_neglog_q_derivatives_dispatch(
y_p[i],
w_p[i],
q,
mu_p[i],
dmu_p[i],
d2mu_p[i],
d3mu_p[i],
link_kind_p,
);
let r_t = -a * r;
let r_ls = -a * q * s;
PsiTermsRow {
r_t,
r_ls,
dr_t: -b * q_psi * r + a * r * sz,
dr_ls: -(a + q * b) * q_psi,
h_tt: b * r * r,
h_tl: r * (a + q * b),
h_ll: q * (a + q * b),
dh_tt: r * r * (c * q_psi - 2.0 * b * sz),
dh_tl: r * ((2.0 * b + c * q) * q_psi - (a + q * b) * sz),
dh_ll: (a + 3.0 * q * b + q * q * c) * q_psi,
obj: r_t * z_t_p[i] + r_ls * z_ls_p[i],
}
})
.collect();
let mut r_t = Array1::<f64>::zeros(n);
let mut r_ls = Array1::<f64>::zeros(n);
let mut dr_t = Array1::<f64>::zeros(n);
let mut dr_ls = Array1::<f64>::zeros(n);
let mut h_tt = Array1::<f64>::zeros(n);
let mut h_tl = Array1::<f64>::zeros(n);
let mut h_ll = Array1::<f64>::zeros(n);
let mut dh_tt = Array1::<f64>::zeros(n);
let mut dh_tl = Array1::<f64>::zeros(n);
let mut dh_ll = Array1::<f64>::zeros(n);
let mut objective_psi = 0.0_f64;
for (i, row) in rows.into_iter().enumerate() {
r_t[i] = row.r_t;
r_ls[i] = row.r_ls;
dr_t[i] = row.dr_t;
dr_ls[i] = row.dr_ls;
h_tt[i] = row.h_tt;
h_tl[i] = row.h_tl;
h_ll[i] = row.h_ll;
dh_tt[i] = row.dh_tt;
dh_tl[i] = row.dh_tl;
dh_ll[i] = row.dh_ll;
objective_psi += row.obj;
}
let hessian_psi_operator = build_two_block_custom_family_joint_psi_operator_from_actions(
dir_a.x_primary_psi.cloned_first_action(),
dir_a.x_ls_psi.cloned_first_action(),
0..pt,
pt..pt + pls,
x_t,
x_ls,
&h_tt,
&h_tl,
&h_ll,
&dh_tt,
&dh_tl,
&dh_ll,
)?;
let x_t_map = dir_a.x_primary_psi.as_linear_map_ref();
let x_ls_map = dir_a.x_ls_psi.as_linear_map_ref();
let score_t = x_t_map.transpose_mul(r_t.view()) + fast_atv(x_t, &dr_t);
let score_ls = x_ls_map.transpose_mul(r_ls.view()) + fast_atv(x_ls, &dr_ls);
let mut score_psi = Array1::<f64>::zeros(total);
score_psi.slice_mut(s![0..pt]).assign(&score_t);
score_psi.slice_mut(s![pt..pt + pls]).assign(&score_ls);
let hessian_psi = if hessian_psi_operator.is_some() {
Array2::zeros((0, 0))
} else {
let h_tt_block = weighted_crossprod_psi_maps(
x_t_map,
h_tt.view(),
CustomFamilyPsiLinearMapRef::Dense(x_t),
)? + &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_t),
h_tt.view(),
x_t_map,
)? + &xt_diag_x_dense(x_t, &dh_tt)?;
let h_tl_block = weighted_crossprod_psi_maps(
x_t_map,
h_tl.view(),
CustomFamilyPsiLinearMapRef::Dense(x_ls),
)? + &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_t),
h_tl.view(),
x_ls_map,
)? + &xt_diag_y_dense(x_t, &dh_tl, x_ls)?;
let h_ll_block = weighted_crossprod_psi_maps(
x_ls_map,
h_ll.view(),
CustomFamilyPsiLinearMapRef::Dense(x_ls),
)? + &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_ls),
h_ll.view(),
x_ls_map,
)? + &xt_diag_x_dense(x_ls, &dh_ll)?;
let mut hessian_psi = Array2::<f64>::zeros((total, total));
hessian_psi.slice_mut(s![0..pt, 0..pt]).assign(&h_tt_block);
hessian_psi
.slice_mut(s![0..pt, pt..pt + pls])
.assign(&h_tl_block);
hessian_psi
.slice_mut(s![pt..pt + pls, pt..pt + pls])
.assign(&h_ll_block);
mirror_upper_to_lower(&mut hessian_psi);
hessian_psi
};
Ok(Some(crate::custom_family::ExactNewtonJointPsiTerms {
objective_psi,
score_psi,
hessian_psi,
hessian_psi_operator,
}))
}
pub(crate) fn exact_newton_joint_psisecond_order_terms_from_designs(
&self,
block_states: &[ParameterBlockState],
derivative_blocks: &[Vec<crate::custom_family::CustomFamilyBlockPsiDerivative>],
psi_i: usize,
psi_j: usize,
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
) -> Result<Option<crate::custom_family::ExactNewtonJointPsiSecondOrderTerms>, String> {
let Some(dir_i) = self.exact_newton_joint_psi_direction(
block_states,
derivative_blocks,
psi_i,
x_t,
x_ls,
&self.policy,
)?
else {
return Ok(None);
};
let Some(dir_j) = self.exact_newton_joint_psi_direction(
block_states,
derivative_blocks,
psi_j,
x_t,
x_ls,
&self.policy,
)?
else {
return Ok(None);
};
Ok(Some(
self.exact_newton_joint_psisecond_order_terms_from_parts(
block_states,
derivative_blocks,
&dir_i,
&dir_j,
x_t,
x_ls,
)?,
))
}
pub(crate) fn exact_newton_joint_psisecond_order_terms_from_parts(
&self,
block_states: &[ParameterBlockState],
derivative_blocks: &[Vec<crate::custom_family::CustomFamilyBlockPsiDerivative>],
dir_i: &LocationScaleJointPsiDirection,
dir_j: &LocationScaleJointPsiDirection,
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
) -> Result<crate::custom_family::ExactNewtonJointPsiSecondOrderTerms, String> {
let second_drifts = self.exact_newton_joint_psisecond_design_drifts(
block_states,
derivative_blocks,
dir_i,
dir_j,
x_t,
x_ls,
)?;
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
let core = binomial_location_scale_core(
&self.y,
&self.weights,
eta_t,
eta_ls,
None,
&self.link_kind,
)?;
let pt = x_t.ncols();
let pls = x_ls.ncols();
let total = pt + pls;
let x_t_i_map = dir_i.x_primary_psi.as_linear_map_ref();
let x_t_j_map = dir_j.x_primary_psi.as_linear_map_ref();
let x_ls_i_map = dir_i.x_ls_psi.as_linear_map_ref();
let x_ls_j_map = dir_j.x_ls_psi.as_linear_map_ref();
let x_t_ab_map = second_psi_linear_map(
second_drifts.x_primary_ab_action.as_ref(),
second_drifts.x_primary_ab.as_ref(),
n,
pt,
);
let x_ls_ab_map = second_psi_linear_map(
second_drifts.x_ls_ab_action.as_ref(),
second_drifts.x_ls_ab.as_ref(),
n,
pls,
);
let mut r_t = Array1::<f64>::zeros(n);
let mut r_ls = Array1::<f64>::zeros(n);
let mut dr_t_i = Array1::<f64>::zeros(n);
let mut dr_t_j = Array1::<f64>::zeros(n);
let mut dr_ls_i = Array1::<f64>::zeros(n);
let mut dr_ls_j = Array1::<f64>::zeros(n);
let mut d2r_t = Array1::<f64>::zeros(n);
let mut d2r_ls = Array1::<f64>::zeros(n);
let mut h_tt = Array1::<f64>::zeros(n);
let mut h_tl = Array1::<f64>::zeros(n);
let mut h_ll = Array1::<f64>::zeros(n);
let mut dh_tt_i = Array1::<f64>::zeros(n);
let mut dh_tt_j = Array1::<f64>::zeros(n);
let mut dh_tl_i = Array1::<f64>::zeros(n);
let mut dh_tl_j = Array1::<f64>::zeros(n);
let mut dh_ll_i = Array1::<f64>::zeros(n);
let mut dh_ll_j = Array1::<f64>::zeros(n);
let mut d2h_tt = Array1::<f64>::zeros(n);
let mut d2h_tl = Array1::<f64>::zeros(n);
let mut d2h_ll = Array1::<f64>::zeros(n);
let mut objective_psi_psi = 0.0;
struct PsiSecondRow {
pub(crate) r_t: f64,
pub(crate) r_ls: f64,
pub(crate) dr_t_i: f64,
pub(crate) dr_t_j: f64,
pub(crate) dr_ls_i: f64,
pub(crate) dr_ls_j: f64,
pub(crate) d2r_t: f64,
pub(crate) d2r_ls: f64,
pub(crate) h_tt: f64,
pub(crate) h_tl: f64,
pub(crate) h_ll: f64,
pub(crate) dh_tt_i: f64,
pub(crate) dh_tt_j: f64,
pub(crate) dh_tl_i: f64,
pub(crate) dh_tl_j: f64,
pub(crate) dh_ll_i: f64,
pub(crate) dh_ll_j: f64,
pub(crate) d2h_tt: f64,
pub(crate) d2h_tl: f64,
pub(crate) d2h_ll: f64,
pub(crate) objective: f64,
}
let y_p = self.y.as_slice().expect("y must be contiguous");
let w_p = self.weights.as_slice().expect("weights must be contiguous");
let q_p = core.q0.as_slice().expect("q0 must be contiguous");
let sigma_p = core.sigma.as_slice().expect("sigma must be contiguous");
let mu_p = core.mu.as_slice().expect("mu must be contiguous");
let dmu_p = core.dmu_dq.as_slice().expect("dmu_dq must be contiguous");
let d2mu_p = core
.d2mu_dq2
.as_slice()
.expect("d2mu_dq2 must be contiguous");
let d3mu_p = core
.d3mu_dq3
.as_slice()
.expect("d3mu_dq3 must be contiguous");
let z_t_i = dir_i
.z_primary_psi
.as_slice()
.expect("z_t_psi_i must be contiguous");
let z_t_j = dir_j
.z_primary_psi
.as_slice()
.expect("z_t_psi_j must be contiguous");
let z_ls_i = dir_i
.z_ls_psi
.as_slice()
.expect("z_ls_psi_i must be contiguous");
let z_ls_j = dir_j
.z_ls_psi
.as_slice()
.expect("z_ls_psi_j must be contiguous");
let z_t_ab = second_drifts
.z_primary_ab
.as_slice()
.expect("z_t_ab must be contiguous");
let z_ls_ab = second_drifts
.z_ls_ab
.as_slice()
.expect("z_ls_ab must be contiguous");
let link_kind_p = &self.link_kind;
let rows: Result<Vec<PsiSecondRow>, String> = (0..n)
.into_par_iter()
.map(|row| {
let q = q_p[row];
let r = 1.0 / sigma_p[row];
let q_i = -r * z_t_i[row] - q * z_ls_i[row];
let q_j = -r * z_t_j[row] - q * z_ls_j[row];
let q_ij = -r * z_t_ab[row]
+ r * (z_t_i[row] * z_ls_j[row] + z_t_j[row] * z_ls_i[row])
+ q * (z_ls_i[row] * z_ls_j[row] - z_ls_ab[row]);
let (a, b, c) = binomial_neglog_q_derivatives_dispatch(
y_p[row],
w_p[row],
q,
mu_p[row],
dmu_p[row],
d2mu_p[row],
d3mu_p[row],
link_kind_p,
);
let d = binomial_neglog_q_fourth_derivative_dispatch(
y_p[row],
w_p[row],
q,
mu_p[row],
dmu_p[row],
d2mu_p[row],
d3mu_p[row],
link_kind_p,
)?;
let u = a + q * b;
let u_i = (2.0 * b + q * c) * q_i;
let u_j = (2.0 * b + q * c) * q_j;
Ok(PsiSecondRow {
r_t: -a * r,
r_ls: -a * q,
dr_t_i: -b * q_i * r + a * r * z_ls_i[row],
dr_t_j: -b * q_j * r + a * r * z_ls_j[row],
dr_ls_i: -u * q_i,
dr_ls_j: -u * q_j,
d2r_t: r
* (-c * q_i * q_j - b * q_ij + b * (q_i * z_ls_j[row] + q_j * z_ls_i[row])
- a * z_ls_i[row] * z_ls_j[row]
+ a * z_ls_ab[row]),
d2r_ls: -((2.0 * b + q * c) * q_i * q_j + u * q_ij),
h_tt: b * r * r,
h_tl: r * u,
h_ll: q * u,
dh_tt_i: r * r * (c * q_i - 2.0 * b * z_ls_i[row]),
dh_tt_j: r * r * (c * q_j - 2.0 * b * z_ls_j[row]),
dh_tl_i: r * (u_i - u * z_ls_i[row]),
dh_tl_j: r * (u_j - u * z_ls_j[row]),
dh_ll_i: (a + 3.0 * q * b + q * q * c) * q_i,
dh_ll_j: (a + 3.0 * q * b + q * q * c) * q_j,
d2h_tt: r
* r
* (d * q_i * q_j + c * q_ij
- 2.0 * c * (q_j * z_ls_i[row] + q_i * z_ls_j[row])
+ 4.0 * b * z_ls_i[row] * z_ls_j[row]
- 2.0 * b * z_ls_ab[row]),
d2h_tl: r
* (((3.0 * c + q * d) * q_j) * q_i + (2.0 * b + q * c) * q_ij
- (2.0 * b + q * c) * (q_j * z_ls_i[row] + q_i * z_ls_j[row])
+ u * (z_ls_i[row] * z_ls_j[row] - z_ls_ab[row])),
d2h_ll: (4.0 * b + 5.0 * q * c + q * q * d) * q_i * q_j
+ (a + 3.0 * q * b + q * q * c) * q_ij,
objective: a * q_ij + b * q_i * q_j,
})
})
.collect();
for (row, vals) in rows?.into_iter().enumerate() {
r_t[row] = vals.r_t;
r_ls[row] = vals.r_ls;
dr_t_i[row] = vals.dr_t_i;
dr_t_j[row] = vals.dr_t_j;
dr_ls_i[row] = vals.dr_ls_i;
dr_ls_j[row] = vals.dr_ls_j;
d2r_t[row] = vals.d2r_t;
d2r_ls[row] = vals.d2r_ls;
h_tt[row] = vals.h_tt;
h_tl[row] = vals.h_tl;
h_ll[row] = vals.h_ll;
dh_tt_i[row] = vals.dh_tt_i;
dh_tt_j[row] = vals.dh_tt_j;
dh_tl_i[row] = vals.dh_tl_i;
dh_tl_j[row] = vals.dh_tl_j;
dh_ll_i[row] = vals.dh_ll_i;
dh_ll_j[row] = vals.dh_ll_j;
d2h_tt[row] = vals.d2h_tt;
d2h_tl[row] = vals.d2h_tl;
d2h_ll[row] = vals.d2h_ll;
objective_psi_psi += vals.objective;
}
let mut score_psi_psi = Array1::<f64>::zeros(total);
score_psi_psi.slice_mut(s![0..pt]).assign(
&(x_t_ab_map.transpose_mul(r_t.view())
+ x_t_i_map.transpose_mul(dr_t_j.view())
+ x_t_j_map.transpose_mul(dr_t_i.view())
+ fast_atv(x_t, &d2r_t)),
);
score_psi_psi.slice_mut(s![pt..pt + pls]).assign(
&(x_ls_ab_map.transpose_mul(r_ls.view())
+ x_ls_i_map.transpose_mul(dr_ls_j.view())
+ x_ls_j_map.transpose_mul(dr_ls_i.view())
+ fast_atv(x_ls, &d2r_ls)),
);
let h_tt_block = weighted_crossprod_psi_maps(
x_t_ab_map,
h_tt.view(),
CustomFamilyPsiLinearMapRef::Dense(x_t),
)? + &weighted_crossprod_psi_maps(x_t_i_map, h_tt.view(), x_t_j_map)?
+ &weighted_crossprod_psi_maps(x_t_j_map, h_tt.view(), x_t_i_map)?
+ &weighted_crossprod_psi_maps(
x_t_i_map,
dh_tt_j.view(),
CustomFamilyPsiLinearMapRef::Dense(x_t),
)?
+ &weighted_crossprod_psi_maps(
x_t_j_map,
dh_tt_i.view(),
CustomFamilyPsiLinearMapRef::Dense(x_t),
)?
+ &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_t),
dh_tt_i.view(),
x_t_j_map,
)?
+ &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_t),
dh_tt_j.view(),
x_t_i_map,
)?
+ &xt_diag_x_dense(x_t, &d2h_tt)?
+ &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_t),
h_tt.view(),
x_t_ab_map,
)?;
let h_tl_block = weighted_crossprod_psi_maps(
x_t_ab_map,
h_tl.view(),
CustomFamilyPsiLinearMapRef::Dense(x_ls),
)? + &weighted_crossprod_psi_maps(x_t_i_map, h_tl.view(), x_ls_j_map)?
+ &weighted_crossprod_psi_maps(x_t_j_map, h_tl.view(), x_ls_i_map)?
+ &weighted_crossprod_psi_maps(
x_t_i_map,
dh_tl_j.view(),
CustomFamilyPsiLinearMapRef::Dense(x_ls),
)?
+ &weighted_crossprod_psi_maps(
x_t_j_map,
dh_tl_i.view(),
CustomFamilyPsiLinearMapRef::Dense(x_ls),
)?
+ &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_t),
dh_tl_i.view(),
x_ls_j_map,
)?
+ &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_t),
dh_tl_j.view(),
x_ls_i_map,
)?
+ &xt_diag_y_dense(x_t, &d2h_tl, x_ls)?
+ &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_t),
h_tl.view(),
x_ls_ab_map,
)?;
let h_ll_block = weighted_crossprod_psi_maps(
x_ls_ab_map,
h_ll.view(),
CustomFamilyPsiLinearMapRef::Dense(x_ls),
)? + &weighted_crossprod_psi_maps(x_ls_i_map, h_ll.view(), x_ls_j_map)?
+ &weighted_crossprod_psi_maps(x_ls_j_map, h_ll.view(), x_ls_i_map)?
+ &weighted_crossprod_psi_maps(
x_ls_i_map,
dh_ll_j.view(),
CustomFamilyPsiLinearMapRef::Dense(x_ls),
)?
+ &weighted_crossprod_psi_maps(
x_ls_j_map,
dh_ll_i.view(),
CustomFamilyPsiLinearMapRef::Dense(x_ls),
)?
+ &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_ls),
dh_ll_i.view(),
x_ls_j_map,
)?
+ &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_ls),
dh_ll_j.view(),
x_ls_i_map,
)?
+ &xt_diag_x_dense(x_ls, &d2h_ll)?
+ &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_ls),
h_ll.view(),
x_ls_ab_map,
)?;
let mut hessian_psi_psi = Array2::<f64>::zeros((total, total));
hessian_psi_psi
.slice_mut(s![0..pt, 0..pt])
.assign(&h_tt_block);
hessian_psi_psi
.slice_mut(s![0..pt, pt..pt + pls])
.assign(&h_tl_block);
hessian_psi_psi
.slice_mut(s![pt..pt + pls, pt..pt + pls])
.assign(&h_ll_block);
mirror_upper_to_lower(&mut hessian_psi_psi);
Ok(crate::custom_family::ExactNewtonJointPsiSecondOrderTerms {
objective_psi_psi,
score_psi_psi,
hessian_psi_psi,
hessian_psi_psi_operator: None,
})
}
pub(crate) fn exact_newton_joint_psihessian_directional_derivative_from_designs(
&self,
block_states: &[ParameterBlockState],
derivative_blocks: &[Vec<crate::custom_family::CustomFamilyBlockPsiDerivative>],
psi_index: usize,
d_beta_flat: &Array1<f64>,
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
) -> Result<Option<Array2<f64>>, String> {
let Some(dir_a) = self.exact_newton_joint_psi_direction(
block_states,
derivative_blocks,
psi_index,
x_t,
x_ls,
&self.policy,
)?
else {
return Ok(None);
};
Ok(Some(
self.exact_newton_joint_psihessian_directional_derivative_from_parts(
block_states,
&dir_a,
d_beta_flat,
x_t,
x_ls,
)?,
))
}
pub(crate) fn exact_newton_joint_psihessian_directional_derivative_from_parts(
&self,
block_states: &[ParameterBlockState],
dir_a: &LocationScaleJointPsiDirection,
d_beta_flat: &Array1<f64>,
x_t: &Array2<f64>,
x_ls: &Array2<f64>,
) -> Result<Array2<f64>, String> {
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
let core = binomial_location_scale_core(
&self.y,
&self.weights,
eta_t,
eta_ls,
None,
&self.link_kind,
)?;
let pt = x_t.ncols();
let pls = x_ls.ncols();
let total = pt + pls;
if d_beta_flat.len() != total {
return Err(GamlssError::DimensionMismatch { reason: format!(
"BinomialLocationScaleFamily joint psi hessian directional derivative length mismatch: got {}, expected {}",
d_beta_flat.len(),
total
) }.into());
}
let xi_t = fast_av(x_t, &d_beta_flat.slice(s![0..pt]));
let xi_ls = fast_av(x_ls, &d_beta_flat.slice(s![pt..pt + pls]));
let x_t_map = dir_a.x_primary_psi.as_linear_map_ref();
let x_ls_map = dir_a.x_ls_psi.as_linear_map_ref();
let mut dh_tt_u = Array1::<f64>::zeros(n);
let mut dh_tl_u = Array1::<f64>::zeros(n);
let mut dh_ll_u = Array1::<f64>::zeros(n);
let mut h_tt_u = Array1::<f64>::zeros(n);
let mut h_tl_u = Array1::<f64>::zeros(n);
let mut h_ll_u = Array1::<f64>::zeros(n);
for row in 0..n {
let q = core.q0[row];
let r = 1.0 / core.sigma[row];
let s = core.dsigma_deta[row] / core.sigma[row];
let xi_ls_s = s * xi_ls[row];
let z_ls_psi_s = s * dir_a.z_ls_psi[row];
let du = -r * xi_t[row] - q * xi_ls_s;
let q_a = -r * dir_a.z_primary_psi[row] - q * z_ls_psi_s;
let q_au = r * dir_a.z_primary_psi[row] * xi_ls_s - du * z_ls_psi_s;
let (a, b, c) = binomial_neglog_q_derivatives_dispatch(
self.y[row],
self.weights[row],
q,
core.mu[row],
core.dmu_dq[row],
core.d2mu_dq2[row],
core.d3mu_dq3[row],
&self.link_kind,
);
let d = binomial_neglog_q_fourth_derivative_dispatch(
self.y[row],
self.weights[row],
q,
core.mu[row],
core.dmu_dq[row],
core.d2mu_dq2[row],
core.d3mu_dq3[row],
&self.link_kind,
)?;
let u = a + q * b;
h_tt_u[row] = r * r * (c * du - 2.0 * b * xi_ls_s);
h_tl_u[row] = r * ((2.0 * b + q * c) * du - u * xi_ls_s);
h_ll_u[row] = (a + 3.0 * q * b + q * q * c) * du;
dh_tt_u[row] = r
* r
* (d * du * q_a + c * q_au - 2.0 * c * (q_a * xi_ls_s + du * z_ls_psi_s)
+ 4.0 * b * xi_ls_s * z_ls_psi_s);
dh_tl_u[row] = r
* (((3.0 * c + q * d) * q_a) * du + (2.0 * b + q * c) * q_au
- (2.0 * b + q * c) * (q_a * xi_ls_s + du * z_ls_psi_s)
+ u * xi_ls_s * z_ls_psi_s);
dh_ll_u[row] = (4.0 * b + 5.0 * q * c + q * q * d) * du * q_a
+ (a + 3.0 * q * b + q * q * c) * q_au;
}
let tt_block = weighted_crossprod_psi_maps(
x_t_map,
h_tt_u.view(),
CustomFamilyPsiLinearMapRef::Dense(x_t),
)? + &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_t),
h_tt_u.view(),
x_t_map,
)? + &xt_diag_x_dense(x_t, &dh_tt_u)?;
let tl_block = weighted_crossprod_psi_maps(
x_t_map,
h_tl_u.view(),
CustomFamilyPsiLinearMapRef::Dense(x_ls),
)? + &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_t),
h_tl_u.view(),
x_ls_map,
)? + &xt_diag_y_dense(x_t, &dh_tl_u, x_ls)?;
let ll_block = weighted_crossprod_psi_maps(
x_ls_map,
h_ll_u.view(),
CustomFamilyPsiLinearMapRef::Dense(x_ls),
)? + &weighted_crossprod_psi_maps(
CustomFamilyPsiLinearMapRef::Dense(x_ls),
h_ll_u.view(),
x_ls_map,
)? + &xt_diag_x_dense(x_ls, &dh_ll_u)?;
let mut out = Array2::<f64>::zeros((total, total));
out.slice_mut(s![0..pt, 0..pt]).assign(&tt_block);
out.slice_mut(s![0..pt, pt..pt + pls]).assign(&tl_block);
out.slice_mut(s![pt..pt + pls, pt..pt + pls])
.assign(&ll_block);
mirror_upper_to_lower(&mut out);
Ok(out)
}
pub fn block_effective_jacobian(
specs: &[ParameterBlockSpec],
block_idx: usize,
) -> Result<Box<dyn BlockEffectiveJacobian>, String> {
crate::families::block_layout::block_jacobian::AdditiveWiggleBlockLayout {
family: "BinomialLocationScaleFamily",
n_outputs: 2,
additive_blocks: &[Self::BLOCK_T, Self::BLOCK_LOG_SIGMA],
wiggle_block: None,
}
.block_effective_jacobian(specs, block_idx)
}
}
impl CustomFamily for BinomialLocationScaleFamily {
fn joint_jeffreys_term_required(&self) -> bool {
true
}
fn exact_newton_joint_hessian_beta_dependent(&self) -> bool {
true
}
fn coefficient_hessian_cost(&self, specs: &[ParameterBlockSpec]) -> u64 {
crate::families::location_scale_engine::location_scale_coefficient_hessian_cost(
self.y.len() as u64,
specs,
)
}
fn evaluate(&self, block_states: &[ParameterBlockState]) -> Result<FamilyEvaluation, String> {
validate_block_count::<GamlssError>("BinomialLocationScaleFamily", 2, block_states.len())?;
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
if eta_t.len() != n || eta_ls.len() != n || self.weights.len() != n {
return Err(GamlssError::DimensionMismatch {
reason: "BinomialLocationScaleFamily input size mismatch".to_string(),
}
.into());
}
let core = binomial_location_scale_core(
&self.y,
&self.weights,
eta_t,
eta_ls,
None,
&self.link_kind,
)?;
if !self.exact_joint_supported() {
return Err(
"BinomialLocationScaleFamily requires exact curvature designs; diagonal fallback has been removed"
.to_string(),
);
}
let threshold_design = self.threshold_design.as_ref().ok_or_else(|| {
"BinomialLocationScaleFamily exact path is missing threshold design".to_string()
})?;
let log_sigma_design = self.log_sigma_design.as_ref().ok_or_else(|| {
"BinomialLocationScaleFamily exact path is missing log-sigma design".to_string()
})?;
let mut grad_eta_t_v = vec![0.0_f64; n];
let mut grad_eta_ls_v = vec![0.0_f64; n];
let y_slice_e = self.y.as_slice().expect("y must be contiguous");
let w_slice_e = self.weights.as_slice().expect("weights must be contiguous");
let q0_slice_e = core.q0.as_slice().expect("q0 must be contiguous");
let eta_t_slice_e = eta_t.as_slice().expect("eta_t must be contiguous");
let eta_ls_slice_e = eta_ls.as_slice().expect("eta_ls must be contiguous");
let link_kind_e = &self.link_kind;
let gradient_pairs: Result<Vec<(f64, f64)>, String> = (0..n)
.into_par_iter()
.map(|i| {
let tower = binomial_location_scale_nll_tower(
y_slice_e[i],
w_slice_e[i],
eta_t_slice_e[i],
eta_ls_slice_e[i],
q0_slice_e[i],
core.mu[i],
core.dmu_dq[i],
core.d2mu_dq2[i],
core.d3mu_dq3[i],
link_kind_e,
false,
)?;
Ok((-tower.g[0], -tower.g[1]))
})
.collect();
for (i, (g_t, g_ls)) in gradient_pairs?.into_iter().enumerate() {
grad_eta_t_v[i] = g_t;
grad_eta_ls_v[i] = g_ls;
}
let grad_eta_t = Array1::from_vec(grad_eta_t_v);
let grad_eta_ls = Array1::from_vec(grad_eta_ls_v);
let grad_t = threshold_design.transpose_vector_multiply(&grad_eta_t);
let grad_ls = log_sigma_design.transpose_vector_multiply(&grad_eta_ls);
let (h_tt, h_ll) = self.exact_newton_block_diagonal_hessians_from_design_matrices(
block_states,
threshold_design,
log_sigma_design,
)?;
Ok(FamilyEvaluation {
log_likelihood: core.log_likelihood,
blockworking_sets: vec![
BlockWorkingSet::ExactNewton {
gradient: grad_t,
hessian: SymmetricMatrix::Dense(h_tt),
},
BlockWorkingSet::ExactNewton {
gradient: grad_ls,
hessian: SymmetricMatrix::Dense(h_ll),
},
],
})
}
fn log_likelihood_only(&self, block_states: &[ParameterBlockState]) -> Result<f64, String> {
validate_block_count::<GamlssError>("BinomialLocationScaleFamily", 2, block_states.len())?;
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
if eta_t.len() != n || eta_ls.len() != n || self.weights.len() != n {
return Err(GamlssError::DimensionMismatch {
reason: "BinomialLocationScaleFamily input size mismatch".to_string(),
}
.into());
}
binomial_location_scale_ll_only(
&self.y,
&self.weights,
eta_t,
eta_ls,
None,
&self.link_kind,
)
}
fn log_likelihood_only_with_options(
&self,
block_states: &[ParameterBlockState],
options: &BlockwiseFitOptions,
) -> Result<f64, String> {
let Some(subsample) = options.outer_score_subsample.as_ref() else {
return self.log_likelihood_only(block_states);
};
validate_block_count::<GamlssError>("BinomialLocationScaleFamily", 2, block_states.len())?;
let n = self.y.len();
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
if eta_t.len() != n || eta_ls.len() != n || self.weights.len() != n {
return Err(GamlssError::DimensionMismatch {
reason: "BinomialLocationScaleFamily input size mismatch".to_string(),
}
.into());
}
use rayon::iter::ParallelIterator;
let link_kind = &self.link_kind;
let ll: Result<f64, String> = subsample
.rows
.par_iter()
.try_fold(
|| 0.0_f64,
|acc, row| -> Result<f64, String> {
let i = row.index;
let wi = self.weights[i];
if wi == 0.0 {
return Ok(acc);
}
let SigmaJet1 { sigma, .. } = exp_sigma_jet1_scalar(eta_ls[i]);
let q = binomial_location_scale_q0(eta_t[i], sigma);
let mu = if matches!(link_kind, InverseLink::Standard(StandardLink::Probit)) {
0.5
} else {
let jet = inverse_link_jet_for_inverse_link(link_kind, q).map_err(|e| {
format!("location-scale inverse-link evaluation failed: {e}")
})?;
jet.mu
};
let term =
binomial_location_scale_log_likelihood(self.y[i], wi, q, link_kind, mu)?;
Ok(acc + row.weight * term)
},
)
.try_reduce(|| 0.0_f64, |a, b| Ok(a + b));
ll
}
fn requires_joint_outer_hyper_path(&self) -> bool {
true
}
fn diagonalworking_weights_directional_derivative(
&self,
_: &[ParameterBlockState],
_: usize,
arr: &Array1<f64>,
) -> Result<Option<Array1<f64>>, String> {
assert!(arr.iter().all(|v| !v.is_nan()));
Err(
"BinomialLocationScaleFamily no longer supports diagonal working weights; exact curvature is required"
.to_string(),
)
}
fn exact_newton_joint_psi_terms(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
derivative_blocks: &[Vec<crate::custom_family::CustomFamilyBlockPsiDerivative>],
psi_index: usize,
) -> Result<Option<crate::custom_family::ExactNewtonJointPsiTerms>, String> {
self.exact_newton_joint_psi_terms_for_specs(
block_states,
specs,
derivative_blocks,
psi_index,
)
}
fn exact_newton_joint_psisecond_order_terms(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
derivative_blocks: &[Vec<crate::custom_family::CustomFamilyBlockPsiDerivative>],
psi_i: usize,
psi_j: usize,
) -> Result<Option<crate::custom_family::ExactNewtonJointPsiSecondOrderTerms>, String> {
self.exact_newton_joint_psisecond_order_terms_for_specs(
block_states,
specs,
derivative_blocks,
psi_i,
psi_j,
)
}
fn exact_newton_joint_psihessian_directional_derivative(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
derivative_blocks: &[Vec<crate::custom_family::CustomFamilyBlockPsiDerivative>],
psi_index: usize,
d_beta_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
self.exact_newton_joint_psihessian_directional_derivative_for_specs(
block_states,
specs,
derivative_blocks,
psi_index,
d_beta_flat,
)
}
fn exact_newton_joint_psi_workspace(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
derivative_blocks: &[Vec<crate::custom_family::CustomFamilyBlockPsiDerivative>],
) -> Result<Option<Arc<dyn ExactNewtonJointPsiWorkspace>>, String> {
if !self.exact_joint_supported() {
return Ok(None);
}
Ok(Some(Arc::new(
BinomialLocationScaleExactNewtonJointPsiWorkspace::new(
self.clone(),
block_states.to_vec(),
specs,
derivative_blocks.to_vec(),
)?,
)))
}
fn exact_newton_hessian_directional_derivative(
&self,
block_states: &[ParameterBlockState],
block_idx: usize,
d_beta: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
if !self.exact_joint_supported() {
return Ok(None);
}
let pt = self
.threshold_design
.as_ref()
.ok_or_else(|| {
"BinomialLocationScaleFamily exact path is missing threshold design".to_string()
})?
.ncols();
let pls = self
.log_sigma_design
.as_ref()
.ok_or_else(|| {
"BinomialLocationScaleFamily exact path is missing log-sigma design".to_string()
})?
.ncols();
let total = pt + pls;
let (start, end, joint_direction) = match block_idx {
Self::BLOCK_T => {
if d_beta.len() != pt {
return Err(GamlssError::DimensionMismatch { reason: format!(
"BinomialLocationScaleFamily threshold d_beta length mismatch: got {}, expected {}",
d_beta.len(),
pt
) }.into());
}
let mut dir = Array1::<f64>::zeros(total);
dir.slice_mut(s![0..pt]).assign(d_beta);
(0usize, pt, dir)
}
Self::BLOCK_LOG_SIGMA => {
if d_beta.len() != pls {
return Err(GamlssError::DimensionMismatch { reason: format!(
"BinomialLocationScaleFamily log-sigma d_beta length mismatch: got {}, expected {}",
d_beta.len(),
pls
) }.into());
}
let mut dir = Array1::<f64>::zeros(total);
dir.slice_mut(s![pt..pt + pls]).assign(d_beta);
(pt, pt + pls, dir)
}
_ => return Ok(None),
};
let joint = self
.exact_newton_joint_hessian_directional_derivative(block_states, &joint_direction)?
.ok_or_else(|| {
format!("missing joint exact-newton directional Hessian for block {block_idx}")
})?;
Ok(Some(joint.slice(s![start..end, start..end]).to_owned()))
}
fn exact_newton_joint_hessian(
&self,
block_states: &[ParameterBlockState],
) -> Result<Option<Array2<f64>>, String> {
self.exact_newton_joint_hessian_for_specs(block_states, None)
}
fn has_explicit_joint_hessian(&self) -> bool {
true
}
fn exact_newton_joint_hessian_directional_derivative(
&self,
block_states: &[ParameterBlockState],
d_beta_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
self.exact_newton_joint_hessian_directional_derivative_for_specs(
block_states,
None,
d_beta_flat,
)
}
fn exact_newton_joint_hessiansecond_directional_derivative(
&self,
block_states: &[ParameterBlockState],
d_beta_u_flat: &Array1<f64>,
d_betav_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
self.exact_newton_joint_hessian_second_directional_derivative_for_specs(
block_states,
None,
d_beta_u_flat,
d_betav_flat,
)
}
fn exact_newton_joint_hessian_with_specs(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
) -> Result<Option<Array2<f64>>, String> {
self.exact_newton_joint_hessian_for_specs(block_states, Some(specs))
}
fn exact_newton_joint_hessian_directional_derivative_with_specs(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
d_beta_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
self.exact_newton_joint_hessian_directional_derivative_for_specs(
block_states,
Some(specs),
d_beta_flat,
)
}
fn exact_newton_joint_hessian_second_directional_derivative_with_specs(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
d_beta_u_flat: &Array1<f64>,
d_betav_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
self.exact_newton_joint_hessian_second_directional_derivative_for_specs(
block_states,
Some(specs),
d_beta_u_flat,
d_betav_flat,
)
}
fn joint_jeffreys_information_with_specs(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
) -> Result<Option<Array2<f64>>, String> {
self.expected_joint_information_for_specs(block_states, Some(specs))
}
fn joint_jeffreys_information_directional_derivative_with_specs(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
d_beta_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
self.expected_joint_information_directional_for_specs(
block_states,
Some(specs),
d_beta_flat,
)
}
fn joint_jeffreys_information_second_directional_derivative_with_specs(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
d_beta_u_flat: &Array1<f64>,
d_betav_flat: &Array1<f64>,
) -> Result<Option<Array2<f64>>, String> {
self.expected_joint_information_second_directional_for_specs(
block_states,
Some(specs),
d_beta_u_flat,
d_betav_flat,
)
}
fn joint_jeffreys_information_contracted_trace_hessian_with_specs(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
weight: &Array2<f64>,
) -> Result<Option<Array2<f64>>, String> {
self.expected_joint_contracted_trace_hessian_for_specs(block_states, Some(specs), weight)
}
fn joint_jeffreys_information_contracted_trace_hessian_available(&self) -> bool {
true
}
fn joint_jeffreys_information_matches_observed_hessian(&self) -> bool {
false
}
fn exact_newton_joint_gradient_evaluation(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
) -> Result<Option<ExactNewtonJointGradientEvaluation>, String> {
let Some((x_t, x_ls)) = self.exact_joint_block_designs_owned(Some(specs))? else {
return Ok(None);
};
self.exact_newton_joint_gradient_from_designs(block_states, &x_t, &x_ls)
.map(Some)
}
fn exact_newton_joint_hessian_workspace(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
) -> Result<Option<Arc<dyn ExactNewtonJointHessianWorkspace>>, String> {
let Some((x_t, x_ls)) = self.exact_joint_block_designs_owned(Some(specs))? else {
return Ok(None);
};
let workspace = BinomialLocationScaleHessianWorkspace::new(
self.clone(),
block_states.to_vec(),
x_t,
x_ls,
)?;
Ok(Some(Arc::new(workspace)))
}
fn exact_newton_joint_hessian_workspace_with_options(
&self,
block_states: &[ParameterBlockState],
specs: &[ParameterBlockSpec],
options: &BlockwiseFitOptions,
) -> Result<Option<Arc<dyn ExactNewtonJointHessianWorkspace>>, String> {
let Some((x_t, x_ls)) = self.exact_joint_block_designs_owned(Some(specs))? else {
return Ok(None);
};
let mut workspace = BinomialLocationScaleHessianWorkspace::new(
self.clone(),
block_states.to_vec(),
x_t,
x_ls,
)?;
if let Some(subsample) = options.outer_score_subsample.as_ref() {
workspace.apply_outer_subsample(subsample.rows.as_ref());
}
Ok(Some(Arc::new(workspace)))
}
fn outer_derivative_subsample_capable(&self) -> bool {
true
}
fn inner_coefficient_hessian_hvp_available(&self, specs: &[ParameterBlockSpec]) -> bool {
if specs.len() != 2 {
return false;
}
let n = self.y.len();
specs[Self::BLOCK_T].design.nrows() == n && specs[Self::BLOCK_LOG_SIGMA].design.nrows() == n
}
}
impl CustomFamilyGenerative for BinomialLocationScaleFamily {
fn generativespec(
&self,
block_states: &[ParameterBlockState],
) -> Result<GenerativeSpec, String> {
validate_block_count::<GamlssError>("BinomialLocationScaleFamily", 2, block_states.len())?;
let eta_t = &block_states[Self::BLOCK_T].eta;
let eta_ls = &block_states[Self::BLOCK_LOG_SIGMA].eta;
if eta_t.len() != self.y.len() || eta_ls.len() != self.y.len() {
return Err(GamlssError::DimensionMismatch {
reason: "BinomialLocationScaleFamily generative size mismatch".to_string(),
}
.into());
}
let mean = gamlss_rowwise_map_result(self.y.len(), |i| {
let sigma = exp_sigma_from_eta_scalar(eta_ls[i]);
let q = binomial_location_scale_q0(eta_t[i], sigma);
let jet = inverse_link_jet_for_inverse_link(&self.link_kind, q)
.map_err(|e| format!("location-scale inverse-link evaluation failed: {e}"))?;
Ok(jet.mu)
})?;
Ok(GenerativeSpec {
mean,
noise: NoiseModel::Bernoulli,
})
}
}