Skip to main content

gam_models/gamlss/
builders.rs

1// Real concern-organized submodule of the gamlss family stack.
2// Cross-module items are re-exported flat through the parent (`gamlss.rs`),
3// so `use super::*;` makes the sibling-concern symbols this module references
4// resolve through the parent namespace.
5use super::*;
6
7#[derive(Clone, Copy)]
8pub(crate) struct GamlssLambdaLayout {
9    pub(crate) k_mean: usize,
10    pub(crate) k_noise: usize,
11    pub(crate) kwiggle: usize,
12}
13
14impl GamlssLambdaLayout {
15    pub(crate) fn two_block(k_mean: usize, k_noise: usize) -> Self {
16        Self {
17            k_mean,
18            k_noise,
19            kwiggle: 0,
20        }
21    }
22
23    pub(crate) fn withwiggle(k_mean: usize, k_noise: usize, kwiggle: usize) -> Self {
24        Self {
25            k_mean,
26            k_noise,
27            kwiggle,
28        }
29    }
30
31    pub(crate) fn total(self) -> usize {
32        self.k_mean + self.k_noise + self.kwiggle
33    }
34
35    pub(crate) fn noise_start(self) -> usize {
36        self.k_mean
37    }
38
39    pub(crate) fn noise_end(self) -> usize {
40        self.k_mean + self.k_noise
41    }
42
43    pub(crate) fn wiggle_start(self) -> usize {
44        self.k_mean + self.k_noise
45    }
46
47    pub(crate) fn wiggle_end(self) -> usize {
48        self.k_mean + self.k_noise + self.kwiggle
49    }
50
51    pub(crate) fn validate_theta_len(self, theta_len: usize, context: &str) -> Result<(), String> {
52        let needed = self.total();
53        if theta_len < needed {
54            return Err(GamlssError::DimensionMismatch {
55                reason: format!(
56                    "{context} theta too short: got {}, need at least {}",
57                    theta_len, needed
58                ),
59            }
60            .into());
61        }
62        Ok(())
63    }
64
65    pub(crate) fn mean_from(self, theta: &Array1<f64>) -> Array1<f64> {
66        theta.slice(s![0..self.k_mean]).to_owned()
67    }
68
69    pub(crate) fn noise_from(self, theta: &Array1<f64>) -> Array1<f64> {
70        theta
71            .slice(s![self.noise_start()..self.noise_end()])
72            .to_owned()
73    }
74
75    pub(crate) fn wiggle_from(self, theta: &Array1<f64>) -> Array1<f64> {
76        theta
77            .slice(s![self.wiggle_start()..self.wiggle_end()])
78            .to_owned()
79    }
80}
81
82#[derive(Clone, Copy)]
83pub(crate) struct GamlssBetaLayout {
84    pub(crate) pt: usize,
85    pub(crate) pls: usize,
86    pub(crate) pw: usize,
87}
88
89impl GamlssBetaLayout {
90    pub(crate) fn withwiggle(pt: usize, pls: usize, pw: usize) -> Self {
91        Self { pt, pls, pw }
92    }
93
94    pub(crate) fn total(self) -> usize {
95        self.pt + self.pls + self.pw
96    }
97
98    pub(crate) fn split_three(
99        self,
100        flat: &Array1<f64>,
101        context: &str,
102    ) -> Result<(Array1<f64>, Array1<f64>, Array1<f64>), String> {
103        if flat.len() != self.total() {
104            return Err(GamlssError::DimensionMismatch {
105                reason: format!(
106                    "{context} length mismatch: got {}, expected {}",
107                    flat.len(),
108                    self.total()
109                ),
110            }
111            .into());
112        }
113        Ok((
114            flat.slice(s![0..self.pt]).to_owned(),
115            flat.slice(s![self.pt..self.pt + self.pls]).to_owned(),
116            flat.slice(s![self.pt + self.pls..self.total()]).to_owned(),
117        ))
118    }
119}
120
121#[derive(Clone, Debug)]
122pub struct FamilyMetadata {
123    pub name: &'static str,
124    pub parameternames: &'static [&'static str],
125    pub parameter_links: &'static [ParameterLink],
126}
127
128pub(crate) const DEFAULT_GAUGE_PRIORITY: u8 = 100;
129
130pub(crate) const LINK_WIGGLE_GAUGE_PRIORITY: u8 = 80;
131
132/// The gauge level at which the frozen, observation-space-residualized warp
133/// block yields to the mean block (#2748).
134///
135/// `audit_identifiability` calls a cross-block alias FATAL **only when the two
136/// blocks carry the same priority**, on the stated grounds that "two blocks
137/// contributing the same direction is only *unfittable* when no ordering exists
138/// to pick which one to drop". On the binomial mean link-wiggle path an
139/// ordering plainly exists, and the de-aliasing IS the proof of it: `B⊥` is by
140/// construction the part of `B(η̂)` the mean block does not explain, so a
141/// direction the two still share after that is one the mean block can represent
142/// and the warp is echoing. The mean block stays intact; the warp yields.
143///
144/// Both blocks reached the audit at `DEFAULT_GAUGE_PRIORITY`, so every such
145/// pair was unfittable-by-declaration. Measured on
146/// `geo_disease_eas3_matern_k12`: `FATAL: alias pair 'eta'[11] ~ 'wiggle'[0]
147/// overlap=0.6542 >= halt half-width 0.1739` at **joint rank 23 of 23 columns
148/// and 0 dropped columns** — a full-rank, merely ill-conditioned design refused
149/// outright.
150///
151/// Two properties make the demotion safe rather than merely permissive:
152///
153/// * at full rank nothing is dropped at all — the ordering only decides who
154///   would yield, and the fit proceeds with both blocks whole;
155/// * if a warp column IS dropped, monotonicity survives it. `dq/dη = 1 + Σ_j
156///   β_j B'_j` with `β_w ≥ 0` and `B'` an M-spline basis, so every retained
157///   term is still non-negative and the sum is still `≥ 1`. Dropping a column
158///   from the penalized mean smooth instead would silently change that
159///   smooth's span, which is the worse of the two.
160///
161/// The sibling `GaussianLocationScaleWiggleFamily` assigns
162/// [`LINK_WIGGLE_GAUGE_PRIORITY`] the OTHER way round — mean and log-σ yield to
163/// the wiggle — and that is not an inconsistency: its warp basis is dynamic and
164/// full-width with no residualization establishing precedence, so there the
165/// wiggle is the block that cannot afford to lose a column. Precedence follows
166/// from which block is defined as the residual, and on this path that is the
167/// warp.
168pub(crate) const DEALIASED_WARP_GAUGE_PRIORITY: u8 = LINK_WIGGLE_GAUGE_PRIORITY;
169
170pub(crate) fn initial_log_lambdas_orzeros(
171    block: &ParameterBlockInput,
172) -> Result<Array1<f64>, String> {
173    let k = block.penalties.len();
174    let lambdas = block
175        .initial_log_lambdas
176        .clone()
177        .unwrap_or_else(|| Array1::<f64>::zeros(k));
178    if lambdas.len() != k {
179        return Err(GamlssError::DimensionMismatch {
180            reason: format!(
181                "initial_log_lambdas length mismatch: got {}, expected {}",
182                lambdas.len(),
183                k
184            ),
185        }
186        .into());
187    }
188    gam_problem::validate_log_strengths(lambdas.iter().copied())
189        .map_err(|error| format!("initial_log_lambdas: {error}"))?;
190    Ok(lambdas)
191}
192
193fn fitted_log_lambdas(lambdas: &Array1<f64>, context: &str) -> Result<Array1<f64>, String> {
194    lambdas
195        .iter()
196        .copied()
197        .enumerate()
198        .map(|(coordinate, value)| {
199            gam_problem::checked_log_strength(value)
200                .map_err(|error| format!("{context} coordinate {coordinate}: {error}"))
201        })
202        .collect::<Result<Vec<_>, _>>()
203        .map(Array1::from_vec)
204}
205
206pub(crate) fn build_two_block_exact_joint_setup(
207    data: ArrayView2<'_, f64>,
208    meanspec: &TermCollectionSpec,
209    noisespec: &TermCollectionSpec,
210    mean_penalties: usize,
211    noise_penalties: usize,
212    extra_rho0: &[f64],
213    rho0_override: Option<&Array1<f64>>,
214    kappa_options: &SpatialLengthScaleOptimizationOptions,
215) -> Result<ExactJointHyperSetup, gam_terms::basis::BasisError> {
216    // GAMLSS-specific part: assemble the rho seed in [mean | noise | extra]
217    // penalty order, honoring a caller override when it matches the layout.
218    let rho_dim = mean_penalties + noise_penalties + extra_rho0.len();
219    let mut rho0vec = Array1::<f64>::zeros(rho_dim);
220    if let Some(rho0) = rho0_override.filter(|rho0| rho0.len() == rho_dim) {
221        rho0vec.assign(rho0);
222    } else {
223        for (i, &rho_init) in extra_rho0.iter().enumerate() {
224            rho0vec[mean_penalties + noise_penalties + i] = rho_init;
225        }
226    }
227
228    // Generic part: per-block log(kappa) seed/bounds and exact-joint assembly,
229    // with the two linear predictors (mean, noise) in theta order.
230    build_location_scale_exact_joint_setup(data, &[meanspec, noisespec], rho0vec, kappa_options)
231}
232
233pub(crate) fn gaussian_location_scalewarm_start(
234    y: &Array1<f64>,
235    weights: &Array1<f64>,
236    mu_block: &ParameterBlockSpec,
237    log_sigma_block: &ParameterBlockSpec,
238    ridge_floor: f64,
239    mean_beta_hint: Option<&Array1<f64>>,
240    noise_beta_hint: Option<&Array1<f64>>,
241) -> Result<(Array1<f64>, Array1<f64>, f64), String> {
242    let betamu = if let Some(beta) = mean_beta_hint {
243        beta.clone()
244    } else {
245        solve_penalizedweighted_projection(
246            &mu_block.design,
247            &mu_block.offset,
248            y,
249            weights,
250            &mu_block.penalties,
251            &mu_block.initial_log_lambdas,
252            ridge_floor,
253        )?
254    };
255    let mut mu_hat = mu_block.solver_design().matrixvectormultiply(&betamu);
256    mu_hat += mu_block.solver_offset();
257    let mut weighted_ss = 0.0;
258    let mut weight_sum = 0.0;
259    for i in 0..y.len() {
260        let wi = weights[i].max(0.0);
261        let resid = y[i] - mu_hat[i];
262        weighted_ss += wi * resid * resid;
263        weight_sum += wi;
264    }
265    if !weighted_ss.is_finite() || !weight_sum.is_finite() || weight_sum <= 0.0 {
266        return Err(
267            "gaussian location-scale warm start could not estimate residual scale".to_string(),
268        );
269    }
270    // Warm-start σ̂ must clear the logb floor so the inverse link
271    //   η = log(σ − b)
272    // is finite. Use a relative cushion above b so the warm-start is in the
273    // smooth interior of the link domain.
274    let sigma_hat = (weighted_ss / weight_sum)
275        .sqrt()
276        .max(LOGB_SIGMA_FLOOR * 1.5);
277    let beta_log_sigma = if let Some(beta) = noise_beta_hint {
278        beta.clone()
279    } else {
280        let eta_sigma = (sigma_hat - LOGB_SIGMA_FLOOR).ln();
281        let sigma_target = Array1::from_elem(y.len(), eta_sigma);
282        solve_penalizedweighted_projection(
283            &log_sigma_block.design,
284            &log_sigma_block.offset,
285            &sigma_target,
286            weights,
287            &log_sigma_block.penalties,
288            &log_sigma_block.initial_log_lambdas,
289            ridge_floor,
290        )?
291    };
292    Ok((betamu, beta_log_sigma, sigma_hat))
293}
294
295/// Total output count for every two-block location-scale family in this
296/// module (mu/log_sigma or threshold/log_sigma). The wiggle variants add a
297/// third zero-channel block but still drive only two output channels.
298pub(crate) const LOCATION_SCALE_N_OUTPUTS: usize = 2;
299
300/// Construct a fully wired location-scale parameter block.
301///
302/// This is the **only** way to build a LocationScale `ParameterBlockSpec` in
303/// this module — by construction the `AdditiveBlockJacobian` callback is
304/// always installed, so the channel-aware identifiability audit cannot be
305/// silently bypassed by a future `build_blocks` impl that forgets to wire
306/// the callback at the tail (re-introducing #319).
307///
308/// `own_output` is the zero-based output channel this block drives
309/// (e.g. 0 for `mu`/`threshold`, 1 for `log_sigma`). `n_family_outputs` is
310/// fixed at [`LOCATION_SCALE_N_OUTPUTS`] for every two-block family here
311/// but is exposed so the helper composes cleanly with any future
312/// k-block extension.
313pub(crate) fn build_location_scale_block(
314    name: impl Into<String>,
315    design: DesignMatrix,
316    offset: Array1<f64>,
317    penalties: Vec<PenaltyMatrix>,
318    nullspace_dims: Vec<usize>,
319    initial_log_lambdas: Array1<f64>,
320    initial_beta: Option<Array1<f64>>,
321    own_output: usize,
322    n_family_outputs: usize,
323    caller: &str,
324) -> Result<ParameterBlockSpec, String> {
325    if own_output >= n_family_outputs {
326        return Err(format!(
327            "{caller}: own_output={own_output} >= n_family_outputs={n_family_outputs}"
328        ));
329    }
330    let mut spec = ParameterBlockSpec {
331        name: name.into(),
332        design,
333        offset,
334        penalties,
335        nullspace_dims,
336        initial_log_lambdas,
337        initial_beta,
338        gauge_priority: 100,
339        jacobian_callback: None,
340        stacked_design: None,
341        stacked_offset: None,
342    };
343    let dense = spec.effective_design(caller)?;
344    spec.jacobian_callback = Some(std::sync::Arc::new(AdditiveBlockJacobian {
345        design: dense,
346        own_output,
347        n_family_outputs,
348    }));
349    Ok(spec)
350}
351
352/// Construct the wiggle block that accompanies a two-block location-scale
353/// family. The wiggle modulates the inverse link nonlinearly and
354/// contributes no linear effective Jacobian — the installed callback
355/// therefore exposes a zero `(n × p_w)` design under
356/// `n_family_outputs = LOCATION_SCALE_N_OUTPUTS`.
357pub(crate) fn build_location_scale_wiggle_block(
358    name: impl Into<String>,
359    design: DesignMatrix,
360    offset: Array1<f64>,
361    penalties: Vec<PenaltyMatrix>,
362    nullspace_dims: Vec<usize>,
363    initial_log_lambdas: Array1<f64>,
364    initial_beta: Option<Array1<f64>>,
365    n_rows: usize,
366) -> Result<ParameterBlockSpec, String> {
367    let p_w = design.ncols();
368    let mut spec = ParameterBlockSpec {
369        name: name.into(),
370        design,
371        offset,
372        penalties,
373        nullspace_dims,
374        initial_log_lambdas,
375        initial_beta,
376        gauge_priority: 100,
377        jacobian_callback: None,
378        stacked_design: None,
379        stacked_offset: None,
380    };
381    spec.jacobian_callback = Some(std::sync::Arc::new(AdditiveBlockJacobian {
382        design: ndarray::Array2::<f64>::zeros((n_rows, p_w)),
383        own_output: 0,
384        n_family_outputs: LOCATION_SCALE_N_OUTPUTS,
385    }));
386    Ok(spec)
387}
388
389pub(crate) fn prepared_gaussian_log_sigma_design(
390    mu_design: &DesignMatrix,
391    log_sigma_design: &DesignMatrix,
392) -> Result<DesignMatrix, String> {
393    if mu_design.nrows() != log_sigma_design.nrows() {
394        return Err(GamlssError::DimensionMismatch {
395            reason: format!(
396                "gaussian log-sigma design row mismatch: mean rows={}, log_sigma rows={}",
397                mu_design.nrows(),
398                log_sigma_design.nrows()
399            ),
400        }
401        .into());
402    }
403    // Gaussian location-scale remains identifiable even when μ and log σ use
404    // the same covariate basis:
405    //
406    //   L(μ, η) = 0.5 * Σ_i [ (y_i - μ_i)^2 exp(-2η_i) + 2η_i ],
407    //   μ = X_μ β_μ,  η = X_σ β_σ.
408    //
409    // Shared columns are not a frame mismatch. β_μ and β_σ enter through
410    // different sufficient statistics (residual and residual²), so replacing
411    // X_σ with (I - P_{X_μ}) X_σ would impose an extra constraint and can
412    // erase real heteroscedastic signal when the two blocks share a basis.
413    Ok(log_sigma_design.clone())
414}
415
416pub(crate) fn identified_binomial_log_sigma_design(
417    threshold_design: &TermCollectionDesign,
418    log_sigma_design: &TermCollectionDesign,
419    weights: &Array1<f64>,
420) -> Result<DesignMatrix, String> {
421    let non_intercept_start = log_sigma_design
422        .intercept_range
423        .end
424        .min(log_sigma_design.design.ncols());
425    let transform = build_scale_deviation_transform_design(
426        &threshold_design.design,
427        &log_sigma_design.design,
428        weights,
429        non_intercept_start,
430    )?;
431    build_scale_deviation_operator(
432        threshold_design.design.clone(),
433        log_sigma_design.design.clone(),
434        &transform,
435    )
436}
437
438pub(crate) fn identity_penalty(dim: usize) -> Array2<f64> {
439    let mut penalty = Array2::<f64>::zeros((dim, dim));
440    for i in 0..dim {
441        penalty[[i, i]] = 1.0;
442    }
443    penalty
444}
445
446pub(crate) fn append_binomial_log_sigma_shrinkage_penalty_design(
447    design: &mut TermCollectionDesign,
448) {
449    let p = design.design.ncols();
450    design
451        .penalties
452        .push(BlockwisePenalty::new(0..p, identity_penalty(p)));
453    // Identity penalty penalizes the full space → nullspace dimension is 0.
454    design.nullspace_dims.push(0);
455    design.penaltyinfo.push(PenaltyBlockInfo {
456        global_index: design.penaltyinfo.len(),
457        termname: Some("log_sigma_shrinkage".to_string()),
458        penalty: ActivePenaltyInfo {
459            source: PenaltySource::Other("shrinkage".to_string()),
460            original_index: 0,
461            effective_rank: p,
462            normalization_scale: 1.0,
463            kronecker_factors: None,
464            structural_null_frame: None,
465        },
466    });
467}
468
469/// Build the (mean, log-σ) parameter-block pair for a Gaussian location-scale
470/// family. Shared verbatim by the non-wiggle and wiggle Gaussian builders so the
471/// scale-block construction — prepared log-σ design, formula-native penalties,
472/// and the joint Gaussian warm start — lives in exactly one place. Callers
473/// supply the per-block log-λ vectors sliced from their own layout (two-block vs
474/// with-wiggle) and append any extra blocks.
475pub(crate) fn build_gaussian_mean_and_scale_blocks(
476    y: &Array1<f64>,
477    weights: &Array1<f64>,
478    mean_design: &TermCollectionDesign,
479    noise_design: &TermCollectionDesign,
480    mean_offset: &Array1<f64>,
481    noise_offset: &Array1<f64>,
482    mean_log_lambdas: Array1<f64>,
483    noise_log_lambdas: Array1<f64>,
484    mean_beta_hint: Option<Array1<f64>>,
485    noise_beta_hint: Option<Array1<f64>>,
486    context: &str,
487) -> Result<(ParameterBlockSpec, ParameterBlockSpec), String> {
488    let mean_offset = mean_design
489        .compose_offset(mean_offset.view(), &format!("{context}: mu"))
490        .map_err(|error| error.to_string())?;
491    let noise_offset = noise_design
492        .compose_offset(noise_offset.view(), &format!("{context}: log_sigma"))
493        .map_err(|error| error.to_string())?;
494    let mut meanspec = build_location_scale_block(
495        "mu",
496        mean_design.design.clone(),
497        mean_offset,
498        mean_design.penalties_as_penalty_matrix(),
499        mean_design.nullspace_dims.clone(),
500        mean_log_lambdas,
501        mean_beta_hint,
502        0,
503        LOCATION_SCALE_N_OUTPUTS,
504        &format!("{context}: mu"),
505    )?;
506    let prepared_noise_design =
507        prepared_gaussian_log_sigma_design(&mean_design.design, &noise_design.design)?;
508    // The formula-native penalty topology is authoritative. Smooth terms carry
509    // their own REML-selected null-space penalty when `double_penalty=true`
510    // (the default), while an explicit `double_penalty=false` remains a real
511    // opt-out. In particular the global log-σ intercept is likelihood-identified
512    // and must stay unpenalized: adding a Gaussian-only projector over the joint
513    // null space placed a data-scale-dependent prior on the overall σ level and
514    // introduced an extra smoothing coordinate absent from the formula (#1561).
515    let mut noisespec = build_location_scale_block(
516        "log_sigma",
517        prepared_noise_design,
518        noise_offset,
519        noise_design.penalties_as_penalty_matrix(),
520        noise_design.nullspace_dims.clone(),
521        noise_log_lambdas,
522        noise_beta_hint,
523        1,
524        LOCATION_SCALE_N_OUTPUTS,
525        &format!("{context}: log_sigma"),
526    )?;
527    if meanspec.initial_beta.is_none() || noisespec.initial_beta.is_none() {
528        let (betamu0, beta_ls0, _) = gaussian_location_scalewarm_start(
529            y,
530            weights,
531            &meanspec,
532            &noisespec,
533            1e-10,
534            meanspec.initial_beta.as_ref(),
535            noisespec.initial_beta.as_ref(),
536        )?;
537        if meanspec.initial_beta.is_none() {
538            meanspec.initial_beta = Some(betamu0);
539        }
540        if noisespec.initial_beta.is_none() {
541            noisespec.initial_beta = Some(beta_ls0);
542        }
543    }
544    Ok((meanspec, noisespec))
545}
546
547/// Build the (threshold, log-σ) parameter-block pair for a Binomial
548/// location-scale family. Shared by the non-wiggle and wiggle Binomial builders;
549/// mirrors [`build_gaussian_mean_and_scale_blocks`] but with the binomial-
550/// identified log-σ design, the link-aware joint warm start, and the same
551/// REML-selected full-span scale shrinkage penalty.
552pub(crate) fn build_binomial_threshold_and_scale_blocks(
553    y: &Array1<f64>,
554    weights: &Array1<f64>,
555    link_kind: &InverseLink,
556    mean_design: &TermCollectionDesign,
557    noise_design: &TermCollectionDesign,
558    mean_offset: &Array1<f64>,
559    noise_offset: &Array1<f64>,
560    mean_log_lambdas: Array1<f64>,
561    noise_log_lambdas: Array1<f64>,
562    mean_beta_hint: Option<Array1<f64>>,
563    noise_beta_hint: Option<Array1<f64>>,
564    context: &str,
565) -> Result<(ParameterBlockSpec, ParameterBlockSpec), String> {
566    let mean_offset = mean_design
567        .compose_offset(mean_offset.view(), &format!("{context}: threshold"))
568        .map_err(|error| error.to_string())?;
569    let noise_offset = noise_design
570        .compose_offset(noise_offset.view(), &format!("{context}: log_sigma"))
571        .map_err(|error| error.to_string())?;
572    let identifiednoise_design =
573        identified_binomial_log_sigma_design(mean_design, noise_design, weights)?;
574    let p_noise = identifiednoise_design.ncols();
575    let mut log_sigma_penalty_matrices: Vec<PenaltyMatrix> =
576        noise_design.penalties_as_penalty_matrix();
577    log_sigma_penalty_matrices.push(PenaltyMatrix::Dense(identity_penalty(p_noise)));
578    let mut thresholdspec = build_location_scale_block(
579        "threshold",
580        mean_design.design.clone(),
581        mean_offset,
582        mean_design.penalties_as_penalty_matrix(),
583        vec![],
584        mean_log_lambdas,
585        mean_beta_hint,
586        0,
587        LOCATION_SCALE_N_OUTPUTS,
588        &format!("{context}: threshold"),
589    )?;
590    let mut log_sigmaspec = build_location_scale_block(
591        "log_sigma",
592        identifiednoise_design,
593        noise_offset,
594        log_sigma_penalty_matrices,
595        vec![],
596        noise_log_lambdas,
597        noise_beta_hint,
598        1,
599        LOCATION_SCALE_N_OUTPUTS,
600        &format!("{context}: log_sigma"),
601    )?;
602    if thresholdspec.initial_beta.is_none() || log_sigmaspec.initial_beta.is_none() {
603        let (beta_t0, beta_ls0) = binomial_location_scalewarm_start(
604            y,
605            weights,
606            link_kind,
607            &thresholdspec,
608            &log_sigmaspec,
609            thresholdspec.initial_beta.as_ref(),
610            log_sigmaspec.initial_beta.as_ref(),
611        )?;
612        if thresholdspec.initial_beta.is_none() {
613            thresholdspec.initial_beta = Some(beta_t0);
614        }
615        if log_sigmaspec.initial_beta.is_none() {
616            log_sigmaspec.initial_beta = Some(beta_ls0);
617        }
618    }
619    Ok((thresholdspec, log_sigmaspec))
620}
621
622/// Convert a wiggle block's `PenaltySpec`s into the `PenaltyMatrix` list the
623/// location-scale wiggle block expects. Shared by the Gaussian and Binomial
624/// wiggle builders, which previously inlined the identical match.
625pub(crate) fn wiggle_block_penalty_matrices(
626    wiggle_block: &ParameterBlockInput,
627) -> Vec<PenaltyMatrix> {
628    let p_wiggle = wiggle_block.design.ncols();
629    wiggle_block
630        .penalties
631        .iter()
632        .map(|spec| match spec {
633            crate::model_types::PenaltySpec::Block {
634                local, col_range, ..
635            } => PenaltyMatrix::Blockwise {
636                local: local.clone(),
637                col_range: col_range.clone(),
638                total_dim: p_wiggle,
639            },
640            crate::model_types::PenaltySpec::Dense(m)
641            | crate::model_types::PenaltySpec::DenseWithMean { matrix: m, .. } => {
642                PenaltyMatrix::Dense(m.clone())
643            }
644        })
645        .collect()
646}
647
648pub(crate) fn binomial_location_scale_link_eta_from_probability(
649    link_kind: &InverseLink,
650    probability: f64,
651) -> Result<f64, String> {
652    let target = probability.clamp(1e-6, 1.0 - 1e-6);
653    match link_kind {
654        InverseLink::Standard(StandardLink::Logit) => Ok((target / (1.0 - target)).ln()),
655        InverseLink::Standard(StandardLink::Probit) => standard_normal_quantile(target)
656            .map_err(|err| format!("failed to invert probit warm-start probability: {err}")),
657        InverseLink::Standard(StandardLink::CLogLog) => Ok((-((1.0 - target).ln())).ln()),
658        other => Err(GamlssError::UnsupportedConfiguration { reason: format!(
659            "binomial location-scale warm start requires logit, probit, or cloglog link, got {other:?}"
660        ) }.into()),
661    }
662}
663
664pub(crate) fn weighted_binomial_prevalence(
665    y: &Array1<f64>,
666    weights: &Array1<f64>,
667) -> Result<f64, String> {
668    if y.len() != weights.len() {
669        return Err(GamlssError::DimensionMismatch { reason: format!(
670            "binomial location-scale warm start dimension mismatch: y has length {}, weights have length {}",
671            y.len(),
672            weights.len()
673        ) }.into());
674    }
675    let mut weight_sum = 0.0;
676    let mut success_sum = 0.0;
677    for (&yi, &wi) in y.iter().zip(weights.iter()) {
678        if !yi.is_finite() {
679            return Err(GamlssError::NonFinite {
680                reason: format!(
681                    "binomial location-scale warm start encountered non-finite response {yi}"
682                ),
683            }
684            .into());
685        }
686        if !wi.is_finite() || wi < 0.0 {
687            return Err(GamlssError::InvalidInput {
688                reason: format!(
689                    "binomial location-scale warm start requires finite non-negative weights; weight={wi}"
690                ),
691            }
692            .into());
693        }
694        if wi > 0.0 {
695            weight_sum += wi;
696            success_sum += wi * yi;
697        }
698    }
699    if !weight_sum.is_finite() || weight_sum <= 0.0 {
700        return Err(
701            "binomial location-scale warm start requires positive total weight".to_string(),
702        );
703    }
704    Ok(success_sum / weight_sum)
705}
706
707pub(crate) fn project_constant_eta_into_block(
708    block: &ParameterBlockSpec,
709    weights: &Array1<f64>,
710    eta: f64,
711) -> Result<Array1<f64>, String> {
712    let target_eta = Array1::from_elem(block.design.nrows(), eta);
713    solve_penalizedweighted_projection(
714        &block.design,
715        &block.offset,
716        &target_eta,
717        weights,
718        &block.penalties,
719        &block.initial_log_lambdas,
720        1e-10,
721    )
722}
723
724// Deterministic warm start for the binomial location-scale model. This stays
725// out of the optimizer: it projects a prevalence-matched threshold and neutral
726// log-sigma value into the actual penalized block spaces.
727pub(crate) fn binomial_location_scalewarm_start(
728    y: &Array1<f64>,
729    weights: &Array1<f64>,
730    link_kind: &InverseLink,
731    threshold_block: &ParameterBlockSpec,
732    log_sigma_block: &ParameterBlockSpec,
733    mean_beta_hint: Option<&Array1<f64>>,
734    noise_beta_hint: Option<&Array1<f64>>,
735) -> Result<(Array1<f64>, Array1<f64>), String> {
736    if let (Some(mean_beta), Some(noise_beta)) = (mean_beta_hint, noise_beta_hint) {
737        return Ok((mean_beta.clone(), noise_beta.clone()));
738    }
739
740    let beta_threshold = match mean_beta_hint {
741        Some(beta) => beta.clone(),
742        None => {
743            let prevalence = weighted_binomial_prevalence(y, weights)?;
744            let eta = binomial_location_scale_link_eta_from_probability(link_kind, prevalence)?;
745            project_constant_eta_into_block(threshold_block, weights, eta)?
746        }
747    };
748    let beta_log_sigma = match noise_beta_hint {
749        Some(beta) => beta.clone(),
750        None => project_constant_eta_into_block(log_sigma_block, weights, 0.0)?,
751    };
752    Ok((beta_threshold, beta_log_sigma))
753}
754
755#[derive(Clone)]
756pub(crate) struct BinomialMeanWiggleSpec {
757    pub y: Array1<f64>,
758    pub weights: Array1<f64>,
759    pub link_kind: InverseLink,
760    pub wiggle_knots: Array1<f64>,
761    pub wiggle_degree: usize,
762    pub eta_block: ParameterBlockInput,
763    pub wiggle_block: ParameterBlockInput,
764}
765
766#[derive(Clone)]
767pub struct GaussianLocationScaleTermSpec {
768    pub y: Array1<f64>,
769    pub weights: Array1<f64>,
770    pub meanspec: TermCollectionSpec,
771    pub log_sigmaspec: TermCollectionSpec,
772    pub mean_offset: Array1<f64>,
773    pub log_sigma_offset: Array1<f64>,
774}
775
776#[derive(Clone)]
777pub struct GaussianLocationScaleWiggleTermSpec {
778    pub y: Array1<f64>,
779    pub weights: Array1<f64>,
780    pub meanspec: TermCollectionSpec,
781    pub log_sigmaspec: TermCollectionSpec,
782    pub mean_offset: Array1<f64>,
783    pub log_sigma_offset: Array1<f64>,
784    pub wiggle_knots: Array1<f64>,
785    pub wiggle_degree: usize,
786    pub wiggle_block: ParameterBlockInput,
787}
788
789#[derive(Clone)]
790pub struct BinomialLocationScaleTermSpec {
791    pub y: Array1<f64>,
792    pub weights: Array1<f64>,
793    pub link_kind: InverseLink,
794    pub thresholdspec: TermCollectionSpec,
795    pub log_sigmaspec: TermCollectionSpec,
796    pub threshold_offset: Array1<f64>,
797    pub log_sigma_offset: Array1<f64>,
798}
799
800#[derive(Clone)]
801pub struct BinomialLocationScaleWiggleTermSpec {
802    pub y: Array1<f64>,
803    pub weights: Array1<f64>,
804    pub link_kind: InverseLink,
805    pub thresholdspec: TermCollectionSpec,
806    pub log_sigmaspec: TermCollectionSpec,
807    pub threshold_offset: Array1<f64>,
808    pub log_sigma_offset: Array1<f64>,
809    pub wiggle_knots: Array1<f64>,
810    pub wiggle_degree: usize,
811    pub wiggle_block: ParameterBlockInput,
812}
813
814#[derive(Clone, Debug)]
815pub struct BlockwiseTermFitResult {
816    pub fit: UnifiedFitResult,
817    pub meanspec_resolved: TermCollectionSpec,
818    pub noisespec_resolved: TermCollectionSpec,
819    pub mean_design: TermCollectionDesign,
820    pub noise_design: TermCollectionDesign,
821}
822
823pub(crate) struct BlockwiseTermFitResultParts {
824    pub fit: UnifiedFitResult,
825    pub meanspec_resolved: TermCollectionSpec,
826    pub noisespec_resolved: TermCollectionSpec,
827    pub mean_design: TermCollectionDesign,
828    pub noise_design: TermCollectionDesign,
829}
830
831pub struct BlockwiseTermWiggleFitResult {
832    pub fit: BlockwiseTermFitResult,
833    pub wiggle_knots: Array1<f64>,
834    pub wiggle_degree: usize,
835}
836
837pub struct BinomialMeanWiggleTermFitResult {
838    pub fit: UnifiedFitResult,
839    pub resolvedspec: TermCollectionSpec,
840    pub design: TermCollectionDesign,
841    pub wiggle_knots: Array1<f64>,
842    pub wiggle_degree: usize,
843    /// Standard I-spline warp coefficients `β_w` for the saved-model predict
844    /// runtime when frozen-basis de-aliasing engaged (#1596). Observation-space
845    /// residualization preserves this coefficient chart, so the fit and predict
846    /// runtime consume the same non-negative vector.
847    pub saved_warp_beta: Option<Vec<f64>>,
848    /// Frozen-index mean-coordinate shift `s = β_frozen_source − β_saved` for the
849    /// predict runtime (#2141). Predict evaluates the warp basis at
850    /// `X·(β_saved + s) = η̂` (the frozen index the fit pinned `B` at) instead of
851    /// the de-aliased base predictor `X·β_saved`, reproducing the fitted `q`.
852    pub saved_index_shift: Option<Vec<f64>>,
853}
854
855pub(crate) struct BlockwiseTermWiggleFitResultParts {
856    pub fit: BlockwiseTermFitResult,
857    pub wiggle_knots: Array1<f64>,
858    pub wiggle_degree: usize,
859}
860
861pub(crate) fn validate_term_collection_design(
862    label: &str,
863    design: &TermCollectionDesign,
864) -> Result<(), String> {
865    let p = design.design.ncols();
866    let n = design.design.nrows();
867    for rows in exact_design_row_chunks(n, p) {
868        let chunk = design
869            .design
870            .try_row_chunk(rows)
871            .map_err(|e| format!("{label}.design row chunk materialization failed: {e}"))?;
872        validate_all_finite_estimation(&format!("{label}.design"), chunk.iter().copied())
873            .map_err(|e| e.to_string())?;
874    }
875    if design.nullspace_dims.len() != design.penalties.len() {
876        return Err(GamlssError::DimensionMismatch {
877            reason: format!(
878                "{label}.nullspace_dims length mismatch: got {}, expected {}",
879                design.nullspace_dims.len(),
880                design.penalties.len()
881            ),
882        }
883        .into());
884    }
885    if design.penaltyinfo.len() != design.penalties.len() {
886        return Err(GamlssError::DimensionMismatch {
887            reason: format!(
888                "{label}.penaltyinfo length mismatch: got {}, expected {}",
889                design.penaltyinfo.len(),
890                design.penalties.len()
891            ),
892        }
893        .into());
894    }
895    for (idx, bp) in design.penalties.iter().enumerate() {
896        validate_all_finite_estimation(
897            &format!("{label}.penalties[{idx}]"),
898            bp.local.iter().copied(),
899        )
900        .map_err(|e| e.to_string())?;
901        if bp.col_range.end > p {
902            return Err(GamlssError::DimensionMismatch {
903                reason: format!(
904                    "{label}.penalties[{idx}] col_range {}..{} exceeds design width {}",
905                    bp.col_range.start, bp.col_range.end, p
906                ),
907            }
908            .into());
909        }
910    }
911    if let Some(bounds) = design.coefficient_lower_bounds.as_ref() {
912        if bounds.len() != p {
913            return Err(GamlssError::ConstraintViolation {
914                reason: format!(
915                    "{label}.coefficient_lower_bounds length mismatch: got {}, expected {p}",
916                    bounds.len()
917                ),
918            }
919            .into());
920        }
921        for (idx, &bound) in bounds.iter().enumerate() {
922            if !(bound.is_finite() || bound == f64::NEG_INFINITY) {
923                return Err(GamlssError::NonFinite { reason: format!(
924                    "{label}.coefficient_lower_bounds[{idx}] must be finite or -inf, got {bound}",
925                ) }.into());
926            }
927        }
928    }
929    if let Some(constraints) = design.linear_constraints.as_ref() {
930        validate_all_finite_estimation(
931            &format!("{label}.linear_constraints.a"),
932            constraints.a.iter().copied(),
933        )
934        .map_err(|e| e.to_string())?;
935        validate_all_finite_estimation(
936            &format!("{label}.linear_constraints.b"),
937            constraints.b.iter().copied(),
938        )
939        .map_err(|e| e.to_string())?;
940        if constraints.a.ncols() != p {
941            return Err(GamlssError::DimensionMismatch {
942                reason: format!(
943                    "{label}.linear_constraints.a column mismatch: got {}, expected {p}",
944                    constraints.a.ncols()
945                ),
946            }
947            .into());
948        }
949        if constraints.a.nrows() != constraints.b.len() {
950            return Err(GamlssError::DimensionMismatch {
951                reason: format!(
952                    "{label}.linear_constraints row mismatch: a has {}, b has {}",
953                    constraints.a.nrows(),
954                    constraints.b.len()
955                ),
956            }
957            .into());
958        }
959    }
960    if design.intercept_range.start > design.intercept_range.end || design.intercept_range.end > p {
961        return Err(GamlssError::ConstraintViolation {
962            reason: format!(
963                "{label}.intercept_range out of bounds: {:?} for {} columns",
964                design.intercept_range, p
965            ),
966        }
967        .into());
968    }
969    Ok(())
970}
971
972impl BlockwiseTermFitResult {
973    pub(crate) fn try_from_parts(parts: BlockwiseTermFitResultParts) -> Result<Self, String> {
974        let BlockwiseTermFitResultParts {
975            fit,
976            meanspec_resolved,
977            noisespec_resolved,
978            mean_design,
979            noise_design,
980        } = parts;
981
982        fit.validate_numeric_finiteness()
983            .map_err(|e| format!("{e}"))?;
984        if fit.block_states.len() < 2 {
985            return Err(GamlssError::DimensionMismatch {
986                reason: format!(
987                    "BlockwiseTermFitResult requires at least 2 block states, got {}",
988                    fit.block_states.len()
989                ),
990            }
991            .into());
992        }
993        validate_term_collection_design("blockwise_term.mean_design", &mean_design)?;
994        validate_term_collection_design("blockwise_term.noise_design", &noise_design)?;
995        if mean_design.design.nrows() != noise_design.design.nrows() {
996            return Err(GamlssError::DimensionMismatch {
997                reason: format!(
998                    "BlockwiseTermFitResult row mismatch: mean_design={}, noise_design={}",
999                    mean_design.design.nrows(),
1000                    noise_design.design.nrows()
1001                ),
1002            }
1003            .into());
1004        }
1005        if fit.block_states[0].beta.len() != mean_design.design.ncols() {
1006            return Err(GamlssError::DimensionMismatch {
1007                reason: format!(
1008                    "BlockwiseTermFitResult mean beta length mismatch: got {}, expected {}",
1009                    fit.block_states[0].beta.len(),
1010                    mean_design.design.ncols()
1011                ),
1012            }
1013            .into());
1014        }
1015        if fit.block_states[1].beta.len() != noise_design.design.ncols() {
1016            return Err(GamlssError::DimensionMismatch {
1017                reason: format!(
1018                    "BlockwiseTermFitResult noise beta length mismatch: got {}, expected {}",
1019                    fit.block_states[1].beta.len(),
1020                    noise_design.design.ncols()
1021                ),
1022            }
1023            .into());
1024        }
1025        if fit.block_states[0].eta.len() != mean_design.design.nrows() {
1026            return Err(GamlssError::DimensionMismatch {
1027                reason: format!(
1028                    "BlockwiseTermFitResult mean eta length mismatch: got {}, expected {}",
1029                    fit.block_states[0].eta.len(),
1030                    mean_design.design.nrows()
1031                ),
1032            }
1033            .into());
1034        }
1035        if fit.block_states[1].eta.len() != noise_design.design.nrows() {
1036            return Err(GamlssError::DimensionMismatch {
1037                reason: format!(
1038                    "BlockwiseTermFitResult noise eta length mismatch: got {}, expected {}",
1039                    fit.block_states[1].eta.len(),
1040                    noise_design.design.nrows()
1041                ),
1042            }
1043            .into());
1044        }
1045
1046        Ok(Self {
1047            fit,
1048            meanspec_resolved,
1049            noisespec_resolved,
1050            mean_design,
1051            noise_design,
1052        })
1053    }
1054
1055    pub(crate) fn validate_numeric_finiteness(&self) -> Result<(), String> {
1056        Self::try_from_parts(BlockwiseTermFitResultParts {
1057            fit: self.fit.clone(),
1058            meanspec_resolved: self.meanspec_resolved.clone(),
1059            noisespec_resolved: self.noisespec_resolved.clone(),
1060            mean_design: self.mean_design.clone(),
1061            noise_design: self.noise_design.clone(),
1062        })?;
1063        Ok(())
1064    }
1065}
1066
1067impl BlockwiseTermWiggleFitResult {
1068    pub(crate) fn try_from_parts(parts: BlockwiseTermWiggleFitResultParts) -> Result<Self, String> {
1069        let BlockwiseTermWiggleFitResultParts {
1070            fit,
1071            wiggle_knots,
1072            wiggle_degree,
1073        } = parts;
1074
1075        fit.validate_numeric_finiteness()
1076            .map_err(|e| e.to_string())?;
1077        if fit.fit.block_states.len() < 3 {
1078            return Err(GamlssError::DimensionMismatch {
1079                reason: format!(
1080                    "BlockwiseTermWiggleFitResult requires at least 3 block states, got {}",
1081                    fit.fit.block_states.len()
1082                ),
1083            }
1084            .into());
1085        }
1086        if wiggle_knots.is_empty() {
1087            return Err(GamlssError::UnsupportedConfiguration {
1088                reason: "BlockwiseTermWiggleFitResult requires non-empty wiggle_knots".to_string(),
1089            }
1090            .into());
1091        }
1092        validate_all_finite_estimation(
1093            "blockwise_term_wiggle.wiggle_knots",
1094            wiggle_knots.iter().copied(),
1095        )
1096        .map_err(|e| e.to_string())?;
1097
1098        Ok(Self {
1099            fit,
1100            wiggle_knots,
1101            wiggle_degree,
1102        })
1103    }
1104}
1105
1106pub struct BinomialLocationScaleFitResult {
1107    pub fit: BlockwiseTermFitResult,
1108    pub wiggle_knots: Option<Array1<f64>>,
1109    pub wiggle_degree: Option<usize>,
1110    pub beta_link_wiggle: Option<Vec<f64>>,
1111}
1112
1113pub struct GaussianLocationScaleFitResult {
1114    pub fit: BlockwiseTermFitResult,
1115    pub wiggle_knots: Option<Array1<f64>>,
1116    pub wiggle_degree: Option<usize>,
1117    pub beta_link_wiggle: Option<Vec<f64>>,
1118    /// Response standardization factor applied internally during fitting.
1119    ///
1120    /// The Gaussian location-scale path fits on `y / response_scale` so the
1121    /// fixed log-σ soft floor `LOGB_SIGMA_FLOOR = 0.01` is *operationally*
1122    /// scale-relative (1 % of the response spread) rather than absolute,
1123    /// keeping κ = dlogσ/dη ≈ 1 across the realistic σ range and informing the
1124    /// scale block like gamlss. The returned coefficient `blocks`, `beta`, and
1125    /// link-wiggle knots/coefficients are already mapped back to **raw response
1126    /// units** (the Location/Mean block scaled by `response_scale`, the Scale
1127    /// block intercept shifted by `+ln(response_scale)`), so downstream
1128    /// reconstruction `μ = X_mean·β` comes out in raw units with no further
1129    /// rescaling.
1130    ///
1131    /// The σ reconstruction, however, **must scale the floor too** to stay
1132    /// response-scale-equivariant (#884):
1133    ///
1134    /// ```text
1135    /// σ = response_scale·LOGB_SIGMA_FLOOR + exp(X_scale·β)
1136    ///   = response_scale·(LOGB_SIGMA_FLOOR + exp(η_internal)).
1137    /// ```
1138    ///
1139    /// The intercept shift carries only the `exp(η)` term; reconstructing with a
1140    /// raw `LOGB_SIGMA_FLOOR` instead of `response_scale·LOGB_SIGMA_FLOOR` leaves
1141    /// the non-equivariant residual `LOGB_SIGMA_FLOOR·(1 − response_scale)`.
1142    ///
1143    /// This field records the factor that was applied for transparency,
1144    /// covariance bookkeeping, and the equivariant σ-floor reconstruction; it is
1145    /// `1.0` when no standardization was needed (degenerate constant response).
1146    pub response_scale: f64,
1147}
1148
1149/// Exact coefficient-frame map for the frozen-basis binomial mean-wiggle
1150/// de-aliasing step.
1151///
1152/// The joint solver sees `[X, B - XA]` and returns coordinates
1153/// `(beta_mean_solver, beta_w)`. Saved prediction deliberately uses `[X, B]`,
1154/// so the reported coordinates are
1155///
1156/// ```text
1157/// beta_mean_saved = beta_mean_solver - A beta_w
1158/// beta_w_saved    = beta_w.
1159/// ```
1160///
1161/// This is one linear section with the cross-block lift
1162/// `M = [[I, -A], [0, I]]`. Keeping it as a [`gam_problem::Gauge`] makes the
1163/// same map authoritative for coefficients, covariance, and the active
1164/// geometry lineage used by saved-model ALO.
1165fn binomial_mean_wiggle_saved_frame_gauge(
1166    alias: &Array2<f64>,
1167    mean_width: usize,
1168    wiggle_width: usize,
1169) -> Result<gam_problem::Gauge, String> {
1170    if alias.dim() != (mean_width, wiggle_width) {
1171        return Err(format!(
1172            "binomial mean-wiggle de-alias map is {}x{}, expected {mean_width}x{wiggle_width}",
1173            alias.nrows(),
1174            alias.ncols(),
1175        ));
1176    }
1177    let total_width = mean_width
1178        .checked_add(wiggle_width)
1179        .ok_or_else(|| "binomial mean-wiggle coefficient dimension overflows usize".to_string())?;
1180    let mut transform = Array2::<f64>::eye(total_width);
1181    for row in 0..mean_width {
1182        for column in 0..wiggle_width {
1183            transform[[row, mean_width + column]] = -alias[[row, column]];
1184        }
1185    }
1186    let gauge = gam_problem::Gauge::from_t(
1187        transform,
1188        &[mean_width, wiggle_width],
1189        &[mean_width, wiggle_width],
1190    );
1191    gauge.validate().map_err(|reason| {
1192        format!("binomial mean-wiggle saved coefficient gauge is invalid: {reason}")
1193    })?;
1194    Ok(gauge)
1195}
1196
1197fn binomial_mean_wiggle_saved_geometry(
1198    geometry: &gam_solve::model_types::FitGeometry,
1199    saved_frame: &gam_problem::Gauge,
1200) -> Result<gam_solve::model_types::FitGeometry, String> {
1201    let mut saved_geometry = geometry.clone();
1202    saved_geometry.coefficient_gauge = geometry
1203        .coefficient_gauge
1204        .left_compose(saved_frame)
1205        .map_err(|reason| {
1206            format!(
1207                "binomial mean-wiggle active geometry cannot compose with its exact saved-result gauge: {reason}"
1208            )
1209        })?;
1210    Ok(saved_geometry)
1211}
1212
1213fn binomial_mean_wiggle_saved_covariance(
1214    covariance: &Array2<f64>,
1215    saved_frame: &gam_problem::Gauge,
1216    label: &str,
1217) -> Result<Array2<f64>, String> {
1218    let expected = saved_frame.reduced_total();
1219    if covariance.dim() != (expected, expected) {
1220        return Err(format!(
1221            "binomial mean-wiggle {label} is {}x{}; exact saved-result gauge requires {expected}x{expected} solver-frame coordinates",
1222            covariance.nrows(),
1223            covariance.ncols(),
1224        ));
1225    }
1226    if let Some(((row, column), value)) = covariance
1227        .indexed_iter()
1228        .find(|(_, value)| !value.is_finite())
1229    {
1230        return Err(format!(
1231            "binomial mean-wiggle {label} is non-finite at ({row}, {column}): {value}"
1232        ));
1233    }
1234    let saved = saved_frame.lift_covariance(covariance);
1235    if let Some(((row, column), value)) = saved.indexed_iter().find(|(_, value)| !value.is_finite())
1236    {
1237        return Err(format!(
1238            "binomial mean-wiggle saved-frame {label} is non-finite at ({row}, {column}): {value}"
1239        ));
1240    }
1241    Ok(saved)
1242}
1243
1244/// Atomically move a converged frozen-basis mean-wiggle fit from the solver's
1245/// residualized-design coordinates into the saved prediction coordinates.
1246///
1247/// The penalized Hessian remains in its exact active solver coordinates;
1248/// composing its coefficient gauge records how raw saved rows pull back into
1249/// that frame. Covariances, in contrast, push forward through the saved-frame
1250/// map. No dimension mismatch is ignorable: returning a partially transformed
1251/// fit would make point estimates and uncertainty describe different models.
1252fn finalize_binomial_mean_wiggle_saved_frame(
1253    fit: &mut UnifiedFitResult,
1254    alias: &Array2<f64>,
1255    mean_design: &Array2<f64>,
1256    mean_offset: &Array1<f64>,
1257) -> Result<(), String> {
1258    use gam_problem::BlockRole;
1259
1260    if fit.blocks.len() != 2
1261        || fit.blocks[0].role != BlockRole::Mean
1262        || fit.blocks[1].role != BlockRole::LinkWiggle
1263    {
1264        return Err(format!(
1265            "binomial mean-wiggle saved-frame finalization requires fitted blocks [Mean, LinkWiggle], got {:?}",
1266            fit.blocks
1267                .iter()
1268                .map(|block| block.role)
1269                .collect::<Vec<_>>()
1270        ));
1271    }
1272    if fit.block_states.len() != 2 {
1273        return Err(format!(
1274            "binomial mean-wiggle saved-frame finalization requires two fitted block states, got {}",
1275            fit.block_states.len(),
1276        ));
1277    }
1278    if mean_offset.len() != mean_design.nrows() {
1279        return Err(format!(
1280            "binomial mean-wiggle mean offset has {} rows, expected {}",
1281            mean_offset.len(),
1282            mean_design.nrows(),
1283        ));
1284    }
1285
1286    let mean_width = fit.blocks[0].beta.len();
1287    let wiggle_width = fit.blocks[1].beta.len();
1288    if mean_design.ncols() != mean_width {
1289        return Err(format!(
1290            "binomial mean-wiggle saved mean design has {} columns, expected fitted width {mean_width}",
1291            mean_design.ncols(),
1292        ));
1293    }
1294    for block_index in 0..2 {
1295        if fit.block_states[block_index].beta != fit.blocks[block_index].beta {
1296            return Err(format!(
1297                "binomial mean-wiggle fitted block {block_index} and block-state coefficients disagree before saved-frame finalization"
1298            ));
1299        }
1300    }
1301    let total_width = mean_width
1302        .checked_add(wiggle_width)
1303        .ok_or_else(|| "binomial mean-wiggle coefficient dimension overflows usize".to_string())?;
1304    if fit.beta.len() != total_width {
1305        return Err(format!(
1306            "binomial mean-wiggle flat coefficient vector has width {}, expected {total_width}",
1307            fit.beta.len(),
1308        ));
1309    }
1310    if fit.beta.slice(s![0..mean_width]) != fit.blocks[0].beta
1311        || fit.beta.slice(s![mean_width..total_width]) != fit.blocks[1].beta
1312    {
1313        return Err(
1314            "binomial mean-wiggle flat and block coefficient vectors disagree before saved-frame finalization"
1315                .to_string(),
1316        );
1317    }
1318
1319    let saved_frame = binomial_mean_wiggle_saved_frame_gauge(alias, mean_width, wiggle_width)?;
1320    let saved_blocks =
1321        saved_frame.lift_block_betas(&[fit.blocks[0].beta.clone(), fit.blocks[1].beta.clone()]);
1322    let saved_mean_eta = mean_design.dot(&saved_blocks[0]) + mean_offset;
1323    let mut saved_beta = Array1::<f64>::zeros(total_width);
1324    saved_beta
1325        .slice_mut(s![0..mean_width])
1326        .assign(&saved_blocks[0]);
1327    saved_beta
1328        .slice_mut(s![mean_width..total_width])
1329        .assign(&saved_blocks[1]);
1330
1331    let saved_conditional = fit
1332        .covariance_conditional
1333        .as_ref()
1334        .map(|covariance| {
1335            binomial_mean_wiggle_saved_covariance(
1336                covariance,
1337                &saved_frame,
1338                "conditional covariance",
1339            )
1340        })
1341        .transpose()?;
1342    let saved_corrected = fit
1343        .covariance_corrected
1344        .as_ref()
1345        .map(|covariance| {
1346            binomial_mean_wiggle_saved_covariance(covariance, &saved_frame, "corrected covariance")
1347        })
1348        .transpose()?;
1349    let saved_geometry = binomial_mean_wiggle_saved_geometry(
1350        fit.geometry.as_ref().ok_or_else(|| {
1351            "binomial mean-wiggle fit is missing its exact active geometry".to_string()
1352        })?,
1353        &saved_frame,
1354    )?;
1355
1356    let mut saved_inference = fit.inference.clone();
1357    if let Some(inference) = saved_inference.as_mut() {
1358        if inference.beta_covariance.is_none() && inference.beta_standard_errors.is_some() {
1359            return Err(
1360                "binomial mean-wiggle inference has conditional standard errors without their covariance"
1361                    .to_string(),
1362            );
1363        }
1364        if inference.beta_covariance_corrected.is_none()
1365            && inference.beta_standard_errors_corrected.is_some()
1366        {
1367            return Err(
1368                "binomial mean-wiggle inference has corrected standard errors without their covariance"
1369                    .to_string(),
1370            );
1371        }
1372        if let Some(covariance) = inference.beta_covariance.take() {
1373            let covariance = binomial_mean_wiggle_saved_covariance(
1374                covariance.as_array(),
1375                &saved_frame,
1376                "inference conditional covariance",
1377            )?;
1378            if inference.beta_standard_errors.is_some() {
1379                inference.beta_standard_errors = Some(
1380                    gam_problem::se_from_covariance(&covariance).map_err(|reason| {
1381                        format!(
1382                            "binomial mean-wiggle saved conditional standard errors are invalid: {reason}"
1383                        )
1384                    })?,
1385                );
1386            }
1387            inference.beta_covariance = Some(covariance.into());
1388        }
1389        if let Some(covariance) = inference.beta_covariance_corrected.take() {
1390            let covariance = binomial_mean_wiggle_saved_covariance(
1391                &covariance,
1392                &saved_frame,
1393                "inference corrected covariance",
1394            )?;
1395            if inference.beta_standard_errors_corrected.is_some() {
1396                inference.beta_standard_errors_corrected = Some(
1397                    gam_problem::se_from_covariance(&covariance).map_err(|reason| {
1398                        format!(
1399                            "binomial mean-wiggle saved corrected standard errors are invalid: {reason}"
1400                        )
1401                    })?,
1402                );
1403            }
1404            inference.beta_covariance_corrected = Some(covariance);
1405        }
1406        if let Some(covariance) = inference.beta_covariance_frequentist.take() {
1407            inference.beta_covariance_frequentist = Some(binomial_mean_wiggle_saved_covariance(
1408                &covariance,
1409                &saved_frame,
1410                "frequentist covariance",
1411            )?);
1412        }
1413        if let Some(correction) = inference.smoothing_correction.take() {
1414            inference.smoothing_correction = Some(binomial_mean_wiggle_saved_covariance(
1415                &correction,
1416                &saved_frame,
1417                "smoothing covariance correction",
1418            )?);
1419        }
1420    }
1421
1422    fit.blocks[0].beta = saved_blocks[0].clone();
1423    fit.blocks[1].beta = saved_blocks[1].clone();
1424    fit.block_states[0].beta = saved_blocks[0].clone();
1425    fit.block_states[0].eta = saved_mean_eta;
1426    fit.block_states[1].beta = saved_blocks[1].clone();
1427    fit.beta = saved_beta;
1428    fit.covariance_conditional = saved_conditional;
1429    fit.covariance_corrected = saved_corrected;
1430    fit.geometry = Some(saved_geometry);
1431    fit.inference = saved_inference;
1432    Ok(())
1433}
1434
1435#[cfg(test)]
1436mod binomial_mean_wiggle_saved_frame_tests {
1437    use super::*;
1438    use ndarray::array;
1439
1440    #[test]
1441    fn cross_block_dealias_composes_non_square_geometry_and_pushes_covariance() {
1442        let alias = array![[2.0], [-0.5]];
1443        let saved_frame = binomial_mean_wiggle_saved_frame_gauge(&alias, 2, 1)
1444            .expect("valid cross-block de-alias map");
1445
1446        // The canonical solver retained one of two Mean directions plus the
1447        // LinkWiggle direction: active(2) -> solver raw(3) is rectangular.
1448        let active_to_solver = gam_problem::Gauge::from_t(
1449            array![[1.0, 0.0], [0.0, 0.0], [0.0, 1.0]],
1450            &[2, 1],
1451            &[1, 1],
1452        );
1453        let active_hessian = array![[7.0, 1.5], [1.5, 4.0]];
1454        let geometry = gam_solve::model_types::FitGeometry {
1455            coefficient_gauge: active_to_solver,
1456            penalized_hessian: active_hessian.clone().into(),
1457            constrained_posterior: None,
1458            working: None,
1459        };
1460        let saved_geometry = binomial_mean_wiggle_saved_geometry(&geometry, &saved_frame)
1461            .expect("non-square active geometry composes through saved frame");
1462
1463        assert_eq!(
1464            saved_geometry.coefficient_gauge.t_full,
1465            array![[1.0, -2.0], [0.0, 0.5], [0.0, 1.0]],
1466        );
1467        assert_eq!(
1468            saved_geometry.penalized_hessian.as_array(),
1469            &active_hessian,
1470            "precision stays in the canonical active frame",
1471        );
1472
1473        let solver_covariance = Array2::<f64>::eye(3);
1474        let saved_covariance = binomial_mean_wiggle_saved_covariance(
1475            &solver_covariance,
1476            &saved_frame,
1477            "test covariance",
1478        )
1479        .expect("covariance pushes into saved frame");
1480        assert_eq!(
1481            saved_covariance,
1482            array![[5.0, -1.0, -2.0], [-1.0, 1.25, 0.5], [-2.0, 0.5, 1.0]],
1483            "the -A cross block must alter both Mean variance and Mean/Wiggle covariance",
1484        );
1485    }
1486}
1487
1488/// Residualize a frozen warp basis against the mean block **in the metric the
1489/// frozen joint solve's own Hessian defines** (#2748).
1490///
1491/// Returns `(A, B⊥)` with
1492///
1493/// ```text
1494///     A  = (XᵀWX)⁺ XᵀWB,     B⊥ = B − X·A,     W = diag(curvature),
1495/// ```
1496///
1497/// so that `Xᵀ W B⊥ = 0` on the range the pseudo-inverse retains.
1498///
1499/// # ⚠ The metric is not decoration — it is the whole content of "identifiable"
1500///
1501/// "The part of `B` outside the mean column space" is meaningless until an
1502/// inner product says what *outside* means, and the only inner product the rest
1503/// of this solve uses is the one its own curvature defines. With the warp basis
1504/// frozen, `q = Xβ + B⊥β_w` is linear in `(β, β_w)`, so the joint
1505/// negative-log-likelihood Hessian the fit assembles is EXACTLY
1506///
1507/// ```text
1508///     [X B⊥]ᵀ diag(m₂) [X B⊥] + penalties,      m₂ = ∂²(−ℓ)/∂q²
1509/// ```
1510///
1511/// — see [`BinomialMeanWiggleFamily::bmw_static_hessian_operator`], whose
1512/// frozen arm has `dq_dq0 = 1` and `basis_d1 = 0`, collapsing its four row
1513/// coefficients to `m₂` in every block. Its cross block is `Xᵀ diag(m₂) B⊥`,
1514/// and `W = diag(m₂)` makes that **zero by construction**. The two blocks are
1515/// then exactly decoupled in the curvature the solve reads, which is what
1516/// de-aliasing was for.
1517///
1518/// Taking `W = I` instead orthogonalizes in a metric no part of the problem
1519/// uses. It leaves `Xᵀ diag(m₂) B⊥ ≠ 0`, and the residue is largest exactly
1520/// where binomial data are least informative, because `m₂ = w·μ(1−μ)` on a
1521/// logit spans orders of magnitude across rows near saturation. Two
1522/// consequences:
1523///
1524/// * **The outer frozen-index fixed point stops contracting.** Freeze at `η̂`
1525///   and perturb by `δ`: `B(η̂+δ) ≈ B + diag(δ)B'`, so the design moves by
1526///   `(I−P)diag(s)δ` in `q` with `s = B'β_w`, and the refit answers with
1527///   `Δη = −H(I−P)diag(s)δ`, where `H = X(XᵀWX + S)⁻¹XᵀW` is this fit's own
1528///   hat matrix. With `P` the `W`-projection, `H(I−P) = 0` identically — *for
1529///   any penalty `S`*, since `XᵀW(I−P) = XᵀW − XᵀWX(XᵀWX)⁻¹XᵀW = 0` — so the
1530///   leading term of the outer map's derivative vanishes. With `P` the
1531///   Euclidean projection it does not, and the map then contracts only while
1532///   `max_i s_i` is small, which nothing enforces: monotonicity bounds `β_w`
1533///   below at zero and not at all above.
1534/// * **The `no identifiable warp direction` refusal stops firing on rows that
1535///   carry no information.** An unweighted projection spends its budget
1536///   fitting `B` on saturated rows, where `μ(1−μ) ≈ 0` and the likelihood
1537///   cannot tell `B` from `X` anyway; the `W`-projection leaves those rows in
1538///   the residual, where they belong.
1539///
1540/// # Degenerate metrics
1541///
1542/// A row whose curvature is not a usable weight — non-finite, or negative,
1543/// which a non-canonical link's OBSERVED information can be — must be handed
1544/// in as `0` by the caller. A metric with zero rows is still a valid
1545/// semi-inner product, and the eigen pseudo-inverse below already handles the
1546/// rank deficiency it can produce: directions of `XᵀWX` under the cutoff are
1547/// dropped from `A`, which leaves their (equally negligible) alias in `B⊥`.
1548///
1549/// The cutoff is purely RELATIVE, deliberately. `XᵀWX`'s scale carries the
1550/// curvature's units, which on a binomial logit sit well below one; an
1551/// absolute floor would read the whole spectrum as numerical noise and
1552/// residualize nothing.
1553pub(crate) fn dealias_warp_against_mean_block(
1554    x: &Array2<f64>,
1555    b_full: &Array2<f64>,
1556    curvature: &Array1<f64>,
1557) -> Result<(Array2<f64>, Array2<f64>), String> {
1558    use faer::Side;
1559    use gam_linalg::faer_ndarray::FaerEigh;
1560
1561    let n = x.nrows();
1562    if b_full.nrows() != n || curvature.len() != n {
1563        return Err(format!(
1564            "frozen-basis warp de-aliasing row mismatch: mean block has {n} row(s), warp basis \
1565             has {}, curvature has {}",
1566            b_full.nrows(),
1567            curvature.len()
1568        ));
1569    }
1570    if let Some(bad) = curvature
1571        .iter()
1572        .position(|value| !value.is_finite() || *value < 0.0)
1573    {
1574        return Err(format!(
1575            "frozen-basis warp de-aliasing metric is not a semi-inner product: curvature[{bad}] \
1576             = {}",
1577            curvature[bad]
1578        ));
1579    }
1580    let mut weighted_x = x.clone();
1581    for row in 0..n {
1582        let weight = curvature[row];
1583        weighted_x.row_mut(row).map_inplace(|value| *value *= weight);
1584    }
1585    let xtwx = x.t().dot(&weighted_x);
1586    let xtwb = weighted_x.t().dot(b_full);
1587    let (evals, evecs) = xtwx
1588        .eigh(Side::Lower)
1589        .map_err(|e| format!("frozen-basis warp de-aliasing mean QR failed: {e}"))?;
1590    let max_eval = evals.iter().map(|v| v.abs()).fold(0.0_f64, f64::max);
1591    let cutoff = 1.0e3 * f64::EPSILON * (xtwx.nrows().max(1) as f64) * max_eval;
1592    let mut alias = Array2::<f64>::zeros((x.ncols(), b_full.ncols()));
1593    for k in 0..evals.len() {
1594        let lam = evals[k];
1595        if !lam.is_finite() || lam.abs() <= cutoff {
1596            continue;
1597        }
1598        let uk = evecs.column(k);
1599        let uk_xtwb = uk.t().dot(&xtwb);
1600        for i in 0..alias.nrows() {
1601            for j in 0..alias.ncols() {
1602                alias[[i, j]] += uk[i] * uk_xtwb[j] / lam;
1603            }
1604        }
1605    }
1606    let bda = b_full - &x.dot(&alias);
1607    Ok((alias, bda))
1608}
1609
1610/// The dominant eigenvalue of the frozen-index map's Jacobian, measured from
1611/// two consecutive residuals (#2748).
1612///
1613/// Let `Φ` be the frozen-index map, `η* ` its fixed point, `M = Φ'(η*)` and
1614/// `e_k = η_k − η*`. The undamped iteration is `e_{k+1} = M e_k`, and the
1615/// residual actually available at each pass is `d_k = Φ(η_k) − η_k = (M−I)e_k`.
1616/// `M` commutes with `M − I`, so
1617///
1618/// ```text
1619///     d_k = M d_{k-1}
1620/// ```
1621///
1622/// exactly, and `⟨d_k, d_{k-1}⟩ / ‖d_{k-1}‖²` is `M`'s Rayleigh quotient on a
1623/// vector the power iteration has already aligned with its dominant
1624/// eigenvector. Nothing is chosen: the number is read off two vectors the loop
1625/// computes anyway.
1626///
1627/// Returns `NaN` when there is no previous residual, or when either has zero
1628/// length — a Rayleigh quotient of nothing is not a measurement.
1629pub(crate) fn fixed_point_dominant_multiplier(
1630    previous_residual: Option<&Array1<f64>>,
1631    residual: &Array1<f64>,
1632) -> f64 {
1633    let Some(previous) = previous_residual else {
1634        return f64::NAN;
1635    };
1636    if previous.len() != residual.len() {
1637        return f64::NAN;
1638    }
1639    let denominator = previous.dot(previous);
1640    if !(denominator > 0.0) || !residual.dot(residual).is_finite() {
1641        return f64::NAN;
1642    }
1643    let quotient = residual.dot(previous) / denominator;
1644    if quotient.is_finite() { quotient } else { f64::NAN }
1645}
1646
1647/// The relaxation `t` that removes the dominant mode of a frozen-index
1648/// iteration whose measured multiplier is `mu` (#2748).
1649///
1650/// Relaxing the map to `η + t(Φ(η) − η)` replaces its Jacobian `M` by
1651/// `(1−t)I + tM`, so the dominant mode's multiplier becomes `(1−t) + t·mu`,
1652/// and the `t` that annihilates it is `1/(1 − mu)`. That is Aitken's Δ² for
1653/// this iteration, written as a relaxation.
1654///
1655/// **This function only ever damps.** `1/(1 − mu) ≥ 1` for `mu ∈ [0, 1)`, and
1656/// on that branch it returns `1` — the undamped step, bit for bit. Extrapolating
1657/// a monotone-but-slow mode would be sound for the mode it measures and could
1658/// destabilise a mode of equal magnitude and opposite sign that the Rayleigh
1659/// quotient cannot see, and the failing population here does not need it. So
1660/// the return value is `min(1, 1/(1 − mu))`, i.e.:
1661///
1662/// * `mu < 0` — an ALTERNATING mode, the divergence signature this issue's
1663///   cluster shows: `t = 1/(1 + |mu|) ∈ (0, 1)`, exactly enough damping to
1664///   cancel it. `mu = −4` gives `t = 0.2`.
1665/// * `0 ≤ mu` — `t = 1`. Every pass of every currently-converging fit takes
1666///   this branch, including the first pass of all of them, where `mu` is not
1667///   measurable at all.
1668///
1669/// `mu ≥ 1` is not a case this can repair and does not pretend to be: the
1670/// dominant mode is a monotone divergence, `(1−t) + t·mu ≥ 1` for every `t > 0`,
1671/// and no relaxation of any size stabilises it. The caller reports `mu` in the
1672/// refusal instead.
1673pub(crate) fn frozen_index_relaxation(dominant_multiplier: f64) -> f64 {
1674    if !dominant_multiplier.is_finite() || dominant_multiplier >= 0.0 {
1675        return 1.0;
1676    }
1677    let relaxation = 1.0 / (1.0 - dominant_multiplier);
1678    if relaxation.is_finite() && relaxation > 0.0 {
1679        relaxation
1680    } else {
1681        1.0
1682    }
1683}
1684
1685#[cfg(test)]
1686mod dealiased_warp_gauge_priority_tests {
1687    use super::*;
1688
1689    /// Two blocks that share a direction, at the geometry
1690    /// `geo_disease_eas3_matern_k12` refused on: FULL joint rank, nothing to
1691    /// drop, and a cross-block column overlap far above the leverage-based halt
1692    /// band. At equal priorities `audit_identifiability` calls that FATAL — "no
1693    /// ordering exists to pick which one to drop". Declaring the ordering the
1694    /// de-aliasing already implements is the whole of the repair, so this gates
1695    /// exactly that difference and nothing else: same designs, same overlap,
1696    /// only the priorities move.
1697    fn overlapping_blocks(warp_priority: u8) -> Vec<gam_problem::ParameterBlockSpec> {
1698        let n = 400;
1699        let mut mean = Array2::<f64>::zeros((n, 2));
1700        let mut warp = Array2::<f64>::zeros((n, 2));
1701        for row in 0..n {
1702            let t = row as f64 / (n as f64 - 1.0);
1703            mean[[row, 0]] = 1.0;
1704            mean[[row, 1]] = t;
1705            // ~0.65 overlap with `mean[.., 1]` by construction, plus an
1706            // independent direction so the joint design stays full rank.
1707            warp[[row, 0]] = t + 0.75 * (7.0 * t).sin();
1708            warp[[row, 1]] = (3.0 * t).cos();
1709        }
1710        let mean_block = ParameterBlockInput {
1711            design: DesignMatrix::Dense(gam_linalg::matrix::DenseDesignMatrix::from(mean)),
1712            offset: Array1::zeros(n),
1713            penalties: vec![],
1714            nullspace_dims: vec![],
1715            initial_log_lambdas: Some(Array1::zeros(0)),
1716            initial_beta: None,
1717        };
1718        let warp_block = ParameterBlockInput {
1719            design: DesignMatrix::Dense(gam_linalg::matrix::DenseDesignMatrix::from(warp)),
1720            offset: Array1::zeros(n),
1721            penalties: vec![],
1722            nullspace_dims: vec![],
1723            initial_log_lambdas: Some(Array1::zeros(0)),
1724            initial_beta: None,
1725        };
1726        vec![
1727            mean_block.intospec("eta").expect("eta spec"),
1728            warp_block
1729                .intospec_with_gauge_priority("wiggle", warp_priority)
1730                .expect("wiggle spec"),
1731        ]
1732    }
1733
1734    #[test]
1735    fn equal_priorities_make_a_full_rank_overlap_fatal() {
1736        let specs = overlapping_blocks(DEFAULT_GAUGE_PRIORITY);
1737        let audit = gam_identifiability::audit::audit_identifiability(&specs).expect("audit");
1738        assert!(
1739            audit.dropped_columns.is_empty(),
1740            "the fixture must be full rank so the verdict is about the ORDERING, not a drop: {}",
1741            audit.summary
1742        );
1743        assert!(
1744            audit.fatal,
1745            "the pre-#2748 declaration must still read as unfittable: {}",
1746            audit.summary
1747        );
1748    }
1749
1750    #[test]
1751    fn the_warp_yielding_to_the_mean_block_makes_the_same_overlap_fittable() {
1752        let specs = overlapping_blocks(DEALIASED_WARP_GAUGE_PRIORITY);
1753        let audit = gam_identifiability::audit::audit_identifiability(&specs).expect("audit");
1754        assert!(
1755            audit.dropped_columns.is_empty(),
1756            "declaring an ordering must not itself drop a column at full rank: {}",
1757            audit.summary
1758        );
1759        assert!(
1760            !audit.fatal,
1761            "an ordering exists on this path, so the same overlap is ill-conditioned rather \
1762             than unfittable: {}",
1763            audit.summary
1764        );
1765    }
1766
1767    /// And the ordering points the right way: the warp is the block that yields.
1768    #[test]
1769    fn the_warp_is_the_lower_priority_block() {
1770        assert!(DEALIASED_WARP_GAUGE_PRIORITY < DEFAULT_GAUGE_PRIORITY);
1771    }
1772}
1773
1774#[cfg(test)]
1775mod frozen_index_relaxation_tests {
1776    use super::*;
1777    use ndarray::array;
1778
1779    /// The Rayleigh quotient is exact on a residual pair produced by an actual
1780    /// linear map, which is the situation the derivation claims.
1781    #[test]
1782    fn the_multiplier_recovers_a_planted_linear_map() {
1783        // A diagonal map with a dominant alternating mode.
1784        let previous = array![1.0_f64, 0.0, 0.0];
1785        let residual = array![-3.5_f64, 0.0, 0.0];
1786        let mu = fixed_point_dominant_multiplier(Some(&previous), &residual);
1787        assert!((mu + 3.5).abs() <= 1.0e-14, "mu={mu}");
1788        // And the relaxation it implies annihilates that mode: the relaxed
1789        // multiplier `(1-t) + t*mu` is zero.
1790        let t = frozen_index_relaxation(mu);
1791        assert!(((1.0 - t) + t * mu).abs() <= 1.0e-14, "t={t} mu={mu}");
1792    }
1793
1794    /// A component orthogonal to the previous residual does not contribute to
1795    /// the quotient, which is what makes it an estimate of the DOMINANT mode
1796    /// rather than of the step's length.
1797    #[test]
1798    fn an_orthogonal_component_does_not_move_the_multiplier() {
1799        let previous = array![1.0_f64, 0.0];
1800        let aligned = array![-2.0_f64, 0.0];
1801        let with_orthogonal = array![-2.0_f64, 9.0];
1802        let bare = fixed_point_dominant_multiplier(Some(&previous), &aligned);
1803        let mixed = fixed_point_dominant_multiplier(Some(&previous), &with_orthogonal);
1804        assert!((bare - mixed).abs() <= 1.0e-14, "{bare} vs {mixed}");
1805    }
1806
1807    /// The whole no-regression claim: a non-alternating pass, and the first
1808    /// pass of every fit, take the undamped step bit for bit.
1809    #[test]
1810    fn a_non_alternating_pass_is_the_undamped_step_exactly() {
1811        for mu in [0.0_f64, 0.25, 0.9, 1.0, 4.0, f64::NAN, f64::INFINITY] {
1812            assert_eq!(
1813                frozen_index_relaxation(mu).to_bits(),
1814                1.0_f64.to_bits(),
1815                "mu={mu} must take the undamped branch"
1816            );
1817        }
1818        let residual = array![1.0_f64, -2.0];
1819        assert!(fixed_point_dominant_multiplier(None, &residual).is_nan());
1820        assert_eq!(
1821            frozen_index_relaxation(fixed_point_dominant_multiplier(None, &residual)).to_bits(),
1822            1.0_f64.to_bits(),
1823        );
1824    }
1825
1826    /// Damping never overshoots into a step in the wrong direction and never
1827    /// grows the step: `t` stays in `(0, 1]` for every representable input.
1828    #[test]
1829    fn the_relaxation_never_leaves_the_unit_interval() {
1830        for mu in [
1831            -1.0e300_f64,
1832            -1.0e6,
1833            -12.0,
1834            -1.0,
1835            -1.0e-12,
1836            -0.0,
1837            0.5,
1838            1.0e300,
1839        ] {
1840            let t = frozen_index_relaxation(mu);
1841            assert!(t > 0.0 && t <= 1.0, "mu={mu} gave t={t}");
1842        }
1843    }
1844
1845    /// The measured damping actually turns the archetypal diverging two-cycle
1846    /// into a convergent one: simulate the scalar iteration the derivation
1847    /// models, with a multiplier that makes the undamped map diverge.
1848    #[test]
1849    fn the_derived_relaxation_converges_a_map_the_undamped_iteration_diverges_on() {
1850        let multiplier = -2.5_f64; // |M| > 1: the undamped iteration blows up.
1851        let fixed_point = 0.75_f64;
1852        let map = |eta: f64| fixed_point + multiplier * (eta - fixed_point);
1853
1854        let mut undamped = 1.0_f64;
1855        for _ in 0..40 {
1856            undamped = map(undamped);
1857        }
1858        assert!(
1859            (undamped - fixed_point).abs() > 1.0e6,
1860            "the control must diverge, got {undamped}"
1861        );
1862
1863        let mut eta = 1.0_f64;
1864        let mut previous: Option<Array1<f64>> = None;
1865        for _ in 0..40 {
1866            let residual = Array1::from_elem(1, map(eta) - eta);
1867            let mu = fixed_point_dominant_multiplier(previous.as_ref(), &residual);
1868            let t = frozen_index_relaxation(mu);
1869            previous = Some(residual.clone());
1870            eta += t * residual[0];
1871        }
1872        assert!(
1873            (eta - fixed_point).abs() <= 1.0e-12,
1874            "the damped iteration must reach the fixed point, got {eta}"
1875        );
1876    }
1877}
1878
1879#[cfg(test)]
1880mod binomial_mean_wiggle_dealias_metric_tests {
1881    use super::*;
1882
1883    /// A deterministic mean block, warp basis and curvature vector with the
1884    /// shape the failing cells have: a curvature spanning six orders of
1885    /// magnitude, because that is the whole reason the metric matters
1886    /// (`m₂ = w·μ(1−μ)` on a logit near saturation).
1887    fn fixture() -> (Array2<f64>, Array2<f64>, Array1<f64>) {
1888        let n = 40;
1889        let mut x = Array2::<f64>::zeros((n, 4));
1890        let mut b = Array2::<f64>::zeros((n, 3));
1891        let mut curvature = Array1::<f64>::zeros(n);
1892        for row in 0..n {
1893            let t = row as f64 / (n as f64 - 1.0);
1894            let eta = -6.0 + 12.0 * t;
1895            x[[row, 0]] = 1.0;
1896            x[[row, 1]] = eta;
1897            x[[row, 2]] = (1.7 * eta).sin();
1898            x[[row, 3]] = (0.9 * eta).cos();
1899            // A monotone-ish warp basis: partial sums of positive bumps, which
1900            // is the shape an I-spline has.
1901            b[[row, 0]] = t;
1902            b[[row, 1]] = t * t;
1903            b[[row, 2]] = (1.0 - (-3.0 * t).exp()) / (1.0 - (-3.0_f64).exp());
1904            let mu = 1.0 / (1.0 + (-eta).exp());
1905            curvature[row] = mu * (1.0 - mu);
1906        }
1907        (x, b, curvature)
1908    }
1909
1910    fn cross_block(x: &Array2<f64>, curvature: &Array1<f64>, bda: &Array2<f64>) -> Array2<f64> {
1911        let mut weighted = bda.clone();
1912        for row in 0..bda.nrows() {
1913            let weight = curvature[row];
1914            weighted.row_mut(row).map_inplace(|value| *value *= weight);
1915        }
1916        x.t().dot(&weighted)
1917    }
1918
1919    fn max_abs(matrix: &Array2<f64>) -> f64 {
1920        matrix.iter().map(|value| value.abs()).fold(0.0, f64::max)
1921    }
1922
1923    /// The defining identity: the de-aliased warp block is orthogonal to the
1924    /// mean block in the metric the frozen joint solve's Hessian is assembled
1925    /// in, so the Hessian's cross block is zero.
1926    #[test]
1927    fn the_dealiased_warp_is_curvature_orthogonal_to_the_mean_block() {
1928        let (x, b, curvature) = fixture();
1929        let (_, bda) = dealias_warp_against_mean_block(&x, &b, &curvature).expect("de-alias");
1930        let cross = cross_block(&x, &curvature, &bda);
1931        // Scale the bar by what a single unweighted cross block would have
1932        // been, so this asserts "zero relative to the quantity being cancelled"
1933        // and cannot pass by everything being small.
1934        let reference = max_abs(&cross_block(&x, &curvature, &b));
1935        assert!(reference > 1.0e-3, "the fixture must have a real alias to cancel, got {reference}");
1936        assert!(
1937            max_abs(&cross) <= 1.0e-12 * reference,
1938            "X' W B_perp must vanish: {} against reference {reference}",
1939            max_abs(&cross),
1940        );
1941    }
1942
1943    /// The negative control that makes the point of the fix falsifiable: the
1944    /// EUCLIDEAN residualization — what this site used before #2748 — leaves
1945    /// the solve's own cross block at the same order as the raw alias. It
1946    /// removes nothing the Hessian can see.
1947    #[test]
1948    fn the_euclidean_residualization_leaves_the_solve_coupled() {
1949        let (x, b, curvature) = fixture();
1950        let flat = Array1::<f64>::ones(curvature.len());
1951        let (_, euclidean) = dealias_warp_against_mean_block(&x, &b, &flat).expect("de-alias");
1952        let euclidean_cross = max_abs(&cross_block(&x, &curvature, &euclidean));
1953        let reference = max_abs(&cross_block(&x, &curvature, &b));
1954        assert!(
1955            euclidean_cross >= 0.05 * reference,
1956            "the Euclidean residualization is supposed to leave the curvature cross block \
1957             standing; if this ever becomes small the fixture stopped exercising the defect: \
1958             {euclidean_cross} against {reference}",
1959        );
1960        // And it does cancel the metric it WAS taken in — so the two are
1961        // genuinely different projections, not one of them being broken.
1962        let flat_cross = max_abs(&cross_block(&x, &flat, &euclidean));
1963        let flat_reference = max_abs(&cross_block(&x, &flat, &b));
1964        assert!(
1965            flat_cross <= 1.0e-12 * flat_reference,
1966            "X' B_perp must vanish for the flat metric: {flat_cross} against {flat_reference}",
1967        );
1968    }
1969
1970    /// A constant curvature is the Euclidean projection, exactly. Any fit whose
1971    /// rows are equally informative therefore does not move by this change —
1972    /// which is what makes the change a repair of the ill-conditioned case
1973    /// rather than a new model everywhere.
1974    #[test]
1975    fn a_constant_metric_reproduces_the_euclidean_projection() {
1976        let (x, b, _) = fixture();
1977        let flat = Array1::<f64>::ones(x.nrows());
1978        let scaled = Array1::<f64>::from_elem(x.nrows(), 7.5);
1979        let (alias_flat, _) = dealias_warp_against_mean_block(&x, &b, &flat).expect("flat");
1980        let (alias_scaled, _) = dealias_warp_against_mean_block(&x, &b, &scaled).expect("scaled");
1981        let scale = max_abs(&alias_flat).max(1.0);
1982        assert!(
1983            max_abs(&(&alias_flat - &alias_scaled)) <= 1.0e-12 * scale,
1984            "a metric proportional to the identity is the identity's projection",
1985        );
1986    }
1987
1988    /// Rows the likelihood cannot see get weight zero, and the identity still
1989    /// holds: those rows keep their raw warp in the residual instead of
1990    /// spending the projection's budget on them.
1991    #[test]
1992    fn zero_curvature_rows_leave_their_warp_in_the_residual() {
1993        let (x, b, mut curvature) = fixture();
1994        for row in 0..12 {
1995            curvature[row] = 0.0;
1996        }
1997        let (_, bda) = dealias_warp_against_mean_block(&x, &b, &curvature).expect("de-alias");
1998        assert!(bda.iter().all(|value| value.is_finite()));
1999        let cross = max_abs(&cross_block(&x, &curvature, &bda));
2000        let reference = max_abs(&cross_block(&x, &curvature, &b));
2001        assert!(
2002            cross <= 1.0e-12 * reference,
2003            "X' W B_perp must vanish with a rank-degraded metric too: {cross} against {reference}",
2004        );
2005    }
2006
2007    /// A rank-deficient mean block: the pseudo-inverse drops only its own null
2008    /// directions, and the retained ones are still exactly cancelled.
2009    #[test]
2010    fn a_rank_deficient_mean_block_still_cancels_its_retained_range() {
2011        let (x, b, curvature) = fixture();
2012        let mut duplicated = Array2::<f64>::zeros((x.nrows(), x.ncols() + 1));
2013        duplicated.slice_mut(s![.., ..x.ncols()]).assign(&x);
2014        // An exact copy of column 1, so `X'WX` is singular by construction.
2015        let copied = x.column(1).to_owned();
2016        duplicated.column_mut(x.ncols()).assign(&copied);
2017        let (_, bda) =
2018            dealias_warp_against_mean_block(&duplicated, &b, &curvature).expect("de-alias");
2019        assert!(bda.iter().all(|value| value.is_finite()));
2020        let cross = max_abs(&cross_block(&duplicated, &curvature, &bda));
2021        let reference = max_abs(&cross_block(&duplicated, &curvature, &b));
2022        assert!(
2023            cross <= 1.0e-10 * reference,
2024            "a singular X'WX must not stop the retained range from cancelling: {cross} against \
2025             {reference}",
2026        );
2027    }
2028
2029    /// The mechanism gate, stated on the quantity the outer loop actually
2030    /// rides on rather than on the cross block that implies it (#2748).
2031    ///
2032    /// Freezing the warp basis at `η̂` and perturbing by `δ` moves the design by
2033    /// `(I−P)·diag(s)·δ` in `q`, and the refit answers with
2034    /// `Δη = −H·(I−P)·diag(s)·δ` for this fit's own penalized hat matrix
2035    /// `H = X(XᵀWX + S)⁻¹XᵀW`. That product IS the leading term of the
2036    /// frozen-index map's derivative. With `P` the `W`-projection it is zero
2037    /// for ANY penalty `S`; with `P` Euclidean it is not, and the assertion
2038    /// below is what turns "the metric is the right one" into a measurement.
2039    #[test]
2040    fn the_curvature_metric_annihilates_the_outer_maps_leading_term() {
2041        use faer::Side;
2042        use gam_linalg::faer_ndarray::FaerEigh;
2043
2044        let (x, b, curvature) = fixture();
2045        let n = x.nrows();
2046        let p = x.ncols();
2047        // A warp slope large enough that the undamped map would diverge if the
2048        // term below survived: `max_i s_i` well above one.
2049        let slope = Array1::from_shape_fn(n, |row| 0.4 + 3.0 * (row as f64 / n as f64));
2050        // An arbitrary but deterministic freezing-point perturbation.
2051        let delta = Array1::from_shape_fn(n, |row| ((row as f64) * 0.37).sin());
2052        let forcing = Array1::from_shape_fn(n, |row| slope[row] * delta[row]);
2053
2054        let mut weighted_x = x.clone();
2055        for row in 0..n {
2056            let weight = curvature[row];
2057            weighted_x.row_mut(row).map_inplace(|value| *value *= weight);
2058        }
2059        // A non-trivial ridge penalty, to exercise the "for any S" half of the
2060        // claim: the cancellation must not depend on the smoothing.
2061        let mut normal = x.t().dot(&weighted_x);
2062        for index in 0..p {
2063            normal[[index, index]] += 0.37 * (index as f64 + 1.0);
2064        }
2065        let (normal_values, normal_vectors) = normal.eigh(Side::Lower).expect("penalized normal");
2066        let mut normal_inverse = Array2::<f64>::zeros((p, p));
2067        for k in 0..p {
2068            let scale = 1.0 / normal_values[k];
2069            let uk = normal_vectors.column(k);
2070            for i in 0..p {
2071                for j in 0..p {
2072                    normal_inverse[[i, j]] += scale * uk[i] * uk[j];
2073                }
2074            }
2075        }
2076
2077        // `H·r` for a residual `r`, through the same normal equations the fit
2078        // solves.
2079        let hat_apply = |residual: &Array1<f64>| -> Array1<f64> {
2080            x.dot(&normal_inverse.dot(&weighted_x.t().dot(residual)))
2081        };
2082
2083        let forcing_as_basis = forcing
2084            .clone()
2085            .into_shape_with_order((n, 1))
2086            .expect("column");
2087        let (_, metric_residual) =
2088            dealias_warp_against_mean_block(&x, &forcing_as_basis, &curvature).expect("metric");
2089        let flat = Array1::<f64>::ones(n);
2090        let (_, euclidean_residual) =
2091            dealias_warp_against_mean_block(&x, &forcing_as_basis, &flat).expect("euclidean");
2092
2093        let metric_response = hat_apply(&metric_residual.column(0).to_owned());
2094        let euclidean_response = hat_apply(&euclidean_residual.column(0).to_owned());
2095        let reference = hat_apply(&forcing);
2096        let norm = |v: &Array1<f64>| v.dot(v).sqrt();
2097
2098        assert!(
2099            norm(&reference) > 1.0e-2,
2100            "the fixture must have a leading term to cancel, got {}",
2101            norm(&reference)
2102        );
2103        assert!(
2104            norm(&metric_response) <= 1.0e-12 * norm(&reference),
2105            "H(I-P_W) must annihilate the forcing for any penalty: {} against {}",
2106            norm(&metric_response),
2107            norm(&reference),
2108        );
2109        assert!(
2110            norm(&euclidean_response) >= 0.05 * norm(&reference),
2111            "the Euclidean projector is supposed to leave the outer map's leading term \
2112             standing; if this ever becomes small the fixture stopped exercising the defect: \
2113             {} against {}",
2114            norm(&euclidean_response),
2115            norm(&reference),
2116        );
2117        // And with the leading term gone, the whole B block inherits it: the
2118        // same statement for the real warp basis rather than for one column.
2119        let (_, bda) = dealias_warp_against_mean_block(&x, &b, &curvature).expect("de-alias");
2120        for column in 0..bda.ncols() {
2121            let response = norm(&hat_apply(&bda.column(column).to_owned()));
2122            // Against what the mean block DID absorb from that column before
2123            // residualization — the quantity being cancelled. Grading against
2124            // the residual column's own norm would be grading against a
2125            // different, and smaller, thing.
2126            let absorbed = norm(&hat_apply(&b.column(column).to_owned()));
2127            assert!(
2128                absorbed > 1.0e-3,
2129                "column {column} must have something to cancel, got {absorbed}"
2130            );
2131            assert!(
2132                response <= 1.0e-12 * absorbed,
2133                "column {column}: H B_perp must vanish, got {response} against {absorbed}",
2134            );
2135        }
2136    }
2137
2138    /// A metric that is not a semi-inner product is refused, not silently used:
2139    /// a negative weight would make `X'WX` indefinite and its "projection"
2140    /// would not be one.
2141    #[test]
2142    fn a_negative_curvature_is_refused() {
2143        let (x, b, mut curvature) = fixture();
2144        curvature[7] = -1.0e-3;
2145        let error = dealias_warp_against_mean_block(&x, &b, &curvature)
2146            .expect_err("a negative metric weight must refuse");
2147        assert!(error.contains("semi-inner product"), "{error}");
2148    }
2149}
2150
2151/// Fit the binomial mean link-wiggle model. The observation-space de-aliasing
2152/// preserves the standard I-spline coefficient coordinate, which is returned
2153/// for the saved-model predict runtime.
2154/// What a converged frozen-index binomial mean link-wiggle fit IS, named rather
2155/// than positional.
2156///
2157/// The third and fourth fields exist because the fit is returned in the SAVED
2158/// frame (`finalize_binomial_mean_wiggle_saved_frame`), where the mean
2159/// coefficient has absorbed `−A·β_w`: `q = X·β_saved + B(η̂)·β_w`. So neither
2160/// `block_states[BLOCK_ETA].eta` nor the wiggle block's eta is the frozen index
2161/// the warp was pinned at, and a caller that needs the CRITERION this fit
2162/// realizes cannot reconstruct it from the states alone.
2163///
2164/// `frozen_warp_design` is that criterion's warp block: the accepted `B⊥` the
2165/// fixed point converged with. Returning it means the exact-joint spatial driver
2166/// consumes the very matrix the baseline fit used rather than re-deriving one
2167/// from the saved-frame states — which is how gam#2748's first attempt at this
2168/// went wrong, since `wiggle_design(X·β_saved)` is the warp basis at the
2169/// DE-ALIASED predictor and not at `η̂`.
2170pub(crate) struct BinomialMeanWiggleFrozenFit {
2171    pub(crate) fit: UnifiedFitResult,
2172    pub(crate) saved_warp_beta: Option<Vec<f64>>,
2173    /// `β_frozen_source − β_saved` in mean coordinates, so `X·(β_saved + shift)`
2174    /// is the frozen index `η̂` (#2141).
2175    pub(crate) saved_index_shift: Option<Vec<f64>>,
2176    /// The accepted `B⊥ = B(η̂) − X A`, in the row order of the training data.
2177    pub(crate) frozen_warp_design: std::sync::Arc<Array2<f64>>,
2178}
2179
2180pub(crate) fn fit_binomial_mean_wiggle(
2181    spec: BinomialMeanWiggleSpec,
2182    options: &BlockwiseFitOptions,
2183) -> Result<BinomialMeanWiggleFrozenFit, String> {
2184    let n = spec.y.len();
2185    validate_len_match("weights vs y", n, spec.weights.len())?;
2186    validateweights(&spec.weights, "fit_binomial_mean_wiggle")?;
2187    validate_binomial_response(&spec.y, "fit_binomial_mean_wiggle")?;
2188    validate_blockrows("eta", n, &spec.eta_block)?;
2189    validate_blockrows("wiggle", n, &spec.wiggle_block)?;
2190    if matches!(
2191        spec.link_kind,
2192        InverseLink::Standard(StandardLink::Identity)
2193    ) {
2194        return Err(GamlssError::UnsupportedConfiguration {
2195            reason: "fit_binomial_mean_wiggle does not support identity link".to_string(),
2196        }
2197        .into());
2198    }
2199    gam_terms::inference::formula_dsl::require_binomial_inverse_link_supports_joint_wiggle(
2200        &spec.link_kind,
2201        "fit_binomial_mean_wiggle",
2202    )?;
2203    if spec.wiggle_degree < 2 {
2204        return Err(GamlssError::ConstraintViolation {
2205            reason: format!(
2206                "fit_binomial_mean_wiggle: wiggle_degree must be >= 2, got {}",
2207                spec.wiggle_degree
2208            ),
2209        }
2210        .into());
2211    }
2212    let minimum_knots = minimum_monotone_wiggle_knot_count(spec.wiggle_degree)?;
2213    if spec.wiggle_knots.len() < minimum_knots {
2214        return Err(GamlssError::DimensionMismatch { reason: format!(
2215            "fit_binomial_mean_wiggle: wiggle_knots length {} is too short for degree {} (need at least {})",
2216            spec.wiggle_knots.len(),
2217            spec.wiggle_degree,
2218            minimum_knots
2219        ) }.into());
2220    }
2221
2222    // ----- Frozen-basis Gauss-Newton link-warp fit (#1596) -----
2223    //
2224    // The warp basis `B(η)` is frozen at the current index `η̂` so that
2225    // `q = η + B(η̂)·β_w` is linear in `(β_η, β_w)` (`∂q/∂η = 1`). To keep the
2226    // mean block `X` full and identifiable we fit the warp through the
2227    // observation-space residualized design `B⊥ = (I - P_X)B(η̂)`. We re-freeze
2228    // at the refit `η̂` until the caller's outer convergence policy certifies the
2229    // fixed point.
2230    let x_dense: Array2<f64> = spec.eta_block.design.to_dense();
2231    let (pilot_beta, pilot_eta): (Array1<f64>, Array1<f64>) = {
2232        let pilot_beta = spec.eta_block.initial_beta.clone().ok_or_else(|| {
2233            "fit_binomial_mean_wiggle: eta block carries no pilot β to seed the \
2234             frozen-basis warp index"
2235                .to_string()
2236        })?;
2237        if x_dense.ncols() != pilot_beta.len() {
2238            return Err(GamlssError::DimensionMismatch {
2239                reason: format!(
2240                    "fit_binomial_mean_wiggle: eta design has {} columns but pilot β has {} \
2241                     coefficients",
2242                    x_dense.ncols(),
2243                    pilot_beta.len()
2244                ),
2245            }
2246            .into());
2247        }
2248        let mut eta = x_dense.dot(&pilot_beta);
2249        eta += &spec.eta_block.offset;
2250        (pilot_beta, eta)
2251    };
2252
2253    // Original (full-width) warp penalties / nullspace metadata, captured before
2254    // `spec.wiggle_block` is consumed. The residualized block keeps the same
2255    // coefficient coordinate and therefore the same penalties.
2256    let wiggle_penalties_full = spec.wiggle_block.penalties.clone();
2257    let wiggle_nullspace_dims = spec.wiggle_block.nullspace_dims.clone();
2258    if !wiggle_nullspace_dims.is_empty()
2259        && wiggle_nullspace_dims.len() != wiggle_penalties_full.len()
2260    {
2261        return Err(GamlssError::DimensionMismatch {
2262            reason: format!(
2263                "fit_binomial_mean_wiggle: wiggle block has {} penalties but {} nullspace dimensions",
2264                wiggle_penalties_full.len(),
2265                wiggle_nullspace_dims.len()
2266            ),
2267        }
2268        .into());
2269    }
2270    let wiggle_log_lambdas = spec.wiggle_block.initial_log_lambdas.clone();
2271    let wiggle_beta_initial = spec.wiggle_block.initial_beta.clone();
2272    let eta_block_input = spec.eta_block.clone();
2273
2274    let family = BinomialMeanWiggleFamily {
2275        y: spec.y,
2276        weights: spec.weights,
2277        link_kind: spec.link_kind,
2278        wiggle_knots: spec.wiggle_knots,
2279        wiggle_degree: spec.wiggle_degree,
2280        policy: gam_runtime::resource::ResourcePolicy::default_library(),
2281        frozen_warp_design: None,
2282    };
2283
2284    // Build the de-aliased warp block at a frozen index.  The identifiable
2285    // warp is the part of `B(η̂)` outside the mean column space:
2286    //
2287    //     B⊥ = (I - P_X) B = B - X A,     A = (XᵀWX)^+ XᵀWB.
2288    //
2289    // The previous implementation used `Z = null(XᵀB)` and fitted `B Z`.
2290    // That is too strong: when the warp basis has no coefficient combination
2291    // exactly orthogonal to `X` (for example a two-column flexible link beside
2292    // an intercept+slope mean), it drops every warp coefficient even though
2293    // the nonlinear columns of `B(η̂)` have a nonzero residual after projection
2294    // onto `X`.  Residualizing in observation space removes only the truly
2295    // mean-aliased component and leaves the curved, identifiable link-shape
2296    // signal available to the joint solve.
2297    //
2298    // The returned `A` is used after fitting: because the inner problem used
2299    // `Xβ + (B - XA)β_w`, while prediction reconstructs the saved warp as
2300    // `Xβ_saved + Bβ_w`, we save `β_saved = β - Aβ_w`.
2301    //
2302    // `W` is the curvature `∂²(−ℓ)/∂q²` of the row loss at `working_q`, the
2303    // composite index this fit last realized (the pilot's `η̂` on the first
2304    // pass, where the warp is still zero). The metric is the whole content of
2305    // the word "identifiable" here; see
2306    // [`dealias_warp_against_mean_block`] for why it is that one and not `I`.
2307    let build_dealiased = |frozen: &Array1<f64>,
2308                           working_q: &Array1<f64>,
2309                           beta_hint: Option<&Array1<f64>>,
2310                           log_lambda_hint: Option<&Array1<f64>>|
2311     -> Result<
2312        (
2313            ParameterBlockInput,
2314            Array2<f64>,
2315            std::sync::Arc<Array2<f64>>,
2316        ),
2317        String,
2318    > {
2319        let b_full = family.wiggle_design(frozen.view())?;
2320        let mut curvature = Array1::<f64>::zeros(n);
2321        for row in 0..n {
2322            let (_, m2, _) =
2323                family.neglog_q_derivatives(family.y[row], family.weights[row], working_q[row])?;
2324            curvature[row] = if m2.is_finite() && m2 > 0.0 { m2 } else { 0.0 };
2325        }
2326        let (alias, bda) = dealias_warp_against_mean_block(&x_dense, &b_full, &curvature)?;
2327        let max_b = b_full.iter().map(|v| v.abs()).fold(0.0_f64, f64::max);
2328        let max_resid = bda.iter().map(|v| v.abs()).fold(0.0_f64, f64::max);
2329        let resid_tol =
2330            1.0e3 * f64::EPSILON * (bda.nrows().max(bda.ncols()).max(1) as f64) * max_b.max(1.0);
2331        if max_resid <= resid_tol {
2332            return Err("frozen-basis warp de-aliasing left no identifiable warp \
2333                        direction (the mean block already spans the warp in \
2334                        observation space)"
2335                .to_string());
2336        }
2337        let penalties: Vec<crate::model_types::PenaltySpec> = wiggle_penalties_full
2338            .iter()
2339            .map(|p| {
2340                let s = penalty_spec_to_dense(p, b_full.ncols())?;
2341                Ok(crate::model_types::PenaltySpec::Dense(s))
2342            })
2343            .collect::<Result<_, String>>()?;
2344        let q = bda.ncols();
2345        let initial_beta = match beta_hint {
2346            Some(beta) if beta.len() == q => Some(beta.clone()),
2347            Some(beta) => {
2348                return Err(format!(
2349                    "frozen-basis warp warm start has {} coefficients but the realized basis has {q}",
2350                    beta.len()
2351                ));
2352            }
2353            None => Some(Array1::zeros(q)),
2354        };
2355        let block = ParameterBlockInput {
2356            design: DesignMatrix::Dense(gam_linalg::matrix::DenseDesignMatrix::from(bda.clone())),
2357            offset: Array1::zeros(frozen.len()),
2358            penalties,
2359            nullspace_dims: wiggle_nullspace_dims.clone(),
2360            initial_log_lambdas: log_lambda_hint
2361                .cloned()
2362                .or_else(|| wiggle_log_lambdas.clone()),
2363            initial_beta,
2364        };
2365        Ok((block, alias, std::sync::Arc::new(bda)))
2366    };
2367
2368    // Outer Gauss-Newton / backfitting loop over the frozen warp index. The
2369    // smoothing parameters remain continuously REML/LAML-optimized at every
2370    // step. A fit is returned only after the frozen index is a certified fixed
2371    // point; exhaustion returns non-convergence evidence instead of minting a
2372    // fit from the last iterate. The exact coefficient source used to build the
2373    // accepted frozen basis is retained for prediction (#2141).
2374    if options.outer_max_iter == 0 || !options.outer_tol.is_finite() || options.outer_tol <= 0.0 {
2375        return Err(GamlssError::InvalidInput {
2376            reason: format!(
2377                "fit_binomial_mean_wiggle requires positive outer convergence policy; outer_max_iter={}, outer_tol={}",
2378                options.outer_max_iter, options.outer_tol
2379            ),
2380        }
2381        .into());
2382    }
2383    let mut frozen_source_beta = pilot_beta;
2384    let mut frozen_eta = pilot_eta;
2385    let mut eta_block_warm = eta_block_input.clone();
2386    let mut wiggle_beta_warm = wiggle_beta_initial;
2387    let mut wiggle_log_lambda_warm = wiggle_log_lambdas.clone();
2388    let mut converged: Option<(
2389        UnifiedFitResult,
2390        Array2<f64>,
2391        Array1<f64>,
2392        std::sync::Arc<Array2<f64>>,
2393    )> = None;
2394    let mut last_delta = f64::INFINITY;
2395    let mut last_scale = 1.0_f64;
2396    // #2748 instrumentation. The frozen-basis outer loop is a fixed-point
2397    // iteration on the warp index, and the quantity that decides whether it is
2398    // a contraction is the warp SLOPE `s = B'(eta)*beta_w` — the chain term the
2399    // inner solve drops on purpose (`BinomialMeanWiggleFamily::frozen_warp_design`
2400    // doc). `previous_step` carries the last `Phi(eta_k) - eta_k` so the ratio
2401    // and the alternation angle of successive steps are readable: an
2402    // alternating step whose ratio exceeds one IS the divergence, measured
2403    // rather than inferred from a terminal `delta`.
2404    let mut previous_step: Option<Array1<f64>> = None;
2405    let mut dominant_multiplier = f64::NAN;
2406    // The operating point the de-aliasing metric is evaluated at: the composite
2407    // index `q = X·β + B⊥·β_w` of the previous pass. The pilot carries no warp,
2408    // so on the first pass it is the pilot index itself (#2748).
2409    let mut working_q = frozen_eta.clone();
2410    for _outer in 0..options.outer_max_iter {
2411        let (wiggle_block, alias, bda) = build_dealiased(
2412            &frozen_eta,
2413            &working_q,
2414            wiggle_beta_warm.as_ref(),
2415            wiggle_log_lambda_warm.as_ref(),
2416        )?;
2417        let eta_penalty_count = eta_block_warm.penalties.len();
2418        let wiggle_penalty_count = wiggle_block.penalties.len();
2419        // The warp yields to the mean block when the two still share a
2420        // direction after residualization; see
2421        // [`DEALIASED_WARP_GAUGE_PRIORITY`] for why an ordering exists here at
2422        // all, which is what decides whether such a pair is fatal (#2748).
2423        let blocks = vec![
2424            eta_block_warm.clone().intospec("eta")?,
2425            wiggle_block
2426                .intospec_with_gauge_priority("wiggle", DEALIASED_WARP_GAUGE_PRIORITY)?,
2427        ];
2428        let mut fam = family.clone();
2429        // The pass's own frozen warp block, retained so the accepted pass can
2430        // hand it to the caller: it IS the criterion this fixed point realizes,
2431        // and it is not reconstructible from the saved-frame states (#2748).
2432        let accepted_frozen_warp_design = std::sync::Arc::clone(&bda);
2433        fam.frozen_warp_design = Some(bda);
2434        let fit = fit_custom_family(&fam, &blocks, options).map_err(|e| e.to_string())?;
2435        let mean_state = fit
2436            .block_states
2437            .get(BinomialMeanWiggleFamily::BLOCK_ETA)
2438            .ok_or_else(|| {
2439                "fit_binomial_mean_wiggle: frozen-basis refit did not expose a fitted eta block"
2440                    .to_string()
2441            })?;
2442        if mean_state.eta.len() != frozen_eta.len()
2443            || mean_state.beta.len() != frozen_source_beta.len()
2444        {
2445            return Err(GamlssError::DimensionMismatch {
2446                reason: "fit_binomial_mean_wiggle: frozen-basis refit returned an incompatible eta block"
2447                    .to_string(),
2448            }
2449            .into());
2450        }
2451        let new_eta = mean_state.eta.clone();
2452        let new_source_beta = mean_state.beta.clone();
2453        let wiggle_state = fit
2454            .block_states
2455            .get(BinomialMeanWiggleFamily::BLOCK_WIGGLE)
2456            .ok_or_else(|| {
2457                "fit_binomial_mean_wiggle: frozen-basis refit did not expose a fitted wiggle block"
2458                    .to_string()
2459            })?;
2460        let new_wiggle_beta = wiggle_state.beta.clone();
2461        // The composite index this pass actually fitted, which is the operating
2462        // point the next pass's de-aliasing metric is taken at (#2748).
2463        working_q = &new_eta + &wiggle_state.eta;
2464        last_scale = frozen_eta
2465            .iter()
2466            .chain(new_eta.iter())
2467            .map(|value| value.abs())
2468            .fold(1.0_f64, f64::max);
2469        last_delta = new_eta
2470            .iter()
2471            .zip(frozen_eta.iter())
2472            .map(|(a, b)| (a - b).abs())
2473            .fold(0.0_f64, f64::max);
2474        // #2748: the step, its relation to the previous one, and the warp slope
2475        // that predicts both.
2476        let step = &new_eta - &frozen_eta;
2477        let step_norm = step.dot(&step).sqrt();
2478        let (step_ratio, step_cosine) = match previous_step.as_ref() {
2479            Some(previous) => {
2480                let previous_norm = previous.dot(previous).sqrt();
2481                if previous_norm > 0.0 && step_norm > 0.0 {
2482                    (
2483                        step_norm / previous_norm,
2484                        step.dot(previous) / (step_norm * previous_norm),
2485                    )
2486                } else {
2487                    (f64::NAN, f64::NAN)
2488                }
2489            }
2490            None => (f64::NAN, f64::NAN),
2491        };
2492        dominant_multiplier = fixed_point_dominant_multiplier(previous_step.as_ref(), &step);
2493        let relaxation = frozen_index_relaxation(dominant_multiplier);
2494        let warp_slope = family.wiggle_dq_dq0(frozen_eta.view(), new_wiggle_beta.view())?;
2495        let max_slope = warp_slope
2496            .iter()
2497            .map(|value| value - 1.0)
2498            .fold(f64::NEG_INFINITY, f64::max);
2499        let mean_slope =
2500            warp_slope.iter().map(|value| value - 1.0).sum::<f64>() / warp_slope.len() as f64;
2501        log::info!(
2502            "[WIGGLE-OUTER] #2748 pass {_outer}: delta={last_delta:.6e} scale={last_scale:.6e} \
2503             tol={:.6e} |step|={step_norm:.6e} |step_k|/|step_k-1|={step_ratio:.6e} \
2504             cos(step_k, step_k-1)={step_cosine:+.6} mu_hat={dominant_multiplier:+.6e} \
2505             relaxation={relaxation:.6e} max_warp_slope={max_slope:.6e} \
2506             mean_warp_slope={mean_slope:.6e} |beta_w|_1={:.6e}",
2507            options.outer_tol * last_scale,
2508            new_wiggle_beta.iter().map(|value| value.abs()).sum::<f64>(),
2509        );
2510        previous_step = Some(step.clone());
2511        if last_delta <= options.outer_tol * last_scale {
2512            converged = Some((fit, alias, frozen_source_beta, accepted_frozen_warp_design));
2513            break;
2514        }
2515
2516        let expected_log_lambdas = eta_penalty_count + wiggle_penalty_count;
2517        if fit.log_lambdas.len() != expected_log_lambdas {
2518            return Err(GamlssError::DimensionMismatch {
2519                reason: format!(
2520                    "fit_binomial_mean_wiggle: refit returned {} log-lambdas for {expected_log_lambdas} penalties",
2521                    fit.log_lambdas.len()
2522                ),
2523            }
2524            .into());
2525        }
2526        // Relaxed advance. `relaxation == 1.0` reproduces the undamped
2527        // iteration exactly, which is what every non-oscillating pass gets.
2528        let next_source_beta = &frozen_source_beta
2529            + &((&new_source_beta - &frozen_source_beta).mapv(|value| value * relaxation));
2530        let next_eta = &frozen_eta + &step.mapv(|value| value * relaxation);
2531        eta_block_warm.initial_beta = Some(next_source_beta.clone());
2532        eta_block_warm.initial_log_lambdas =
2533            Some(fit.log_lambdas.slice(s![0..eta_penalty_count]).to_owned());
2534        wiggle_beta_warm = Some(new_wiggle_beta);
2535        wiggle_log_lambda_warm = Some(
2536            fit.log_lambdas
2537                .slice(s![eta_penalty_count..expected_log_lambdas])
2538                .to_owned(),
2539        );
2540        frozen_source_beta = next_source_beta;
2541        frozen_eta = next_eta;
2542    }
2543    let converged = converged.ok_or_else(|| {
2544        GamlssError::NumericalFailure {
2545            reason: format!(
2546                "fit_binomial_mean_wiggle frozen-index fixed point did not converge in {} outer \
2547                 iterations: delta={last_delta:.3e}, scale={last_scale:.3e}, \
2548                 tolerance={:.3e}; the fixed-point map's measured dominant multiplier is \
2549                 mu={dominant_multiplier:.3e} (successive residuals satisfy d_k = M d_(k-1), so \
2550                 this is M's Rayleigh quotient on them) and the relaxation derived from it was \
2551                 {:.3e}. mu >= 1 means the frozen index is a REPELLING fixed point along a \
2552                 monotone direction, which no positive relaxation stabilises: the composite \
2553                 index the warp and the mean block are competing for is not pinned by this \
2554                 model at these smoothing parameters. mu <= -1 with the relaxation at 1 would \
2555                 mean the damping was declined and is a defect here, not a modelling limit",
2556                options.outer_max_iter,
2557                options.outer_tol * last_scale,
2558                frozen_index_relaxation(dominant_multiplier),
2559            ),
2560        }
2561        .to_string()
2562    })?;
2563    let (mut fit, last_alias, frozen_source_beta, frozen_warp_design) = converged;
2564    // Capture the mean coefficients whose linear predictor is the *frozen index*
2565    // `η̂` the warp basis `B(η̂)` was pinned at (#2141). The reported deviance is
2566    // evaluated with `q = X·β_saved + B(η̂)·β_w`, so `predict` must re-evaluate the
2567    // warp basis at `X·β_frozen_source` (= η̂), NOT at the de-aliased base
2568    // predictor `X·β_saved`. On the failing data those differ by the identifiable
2569    // de-alias projection `X·A·β_w`, so predict-at-`β_saved` reconstructs a
2570    // *different* link than the fit used. We persist the shift
2571    // `s = β_frozen_source − β_saved` (a mean-coordinate vector) so predict can form the
2572    // frozen index `X·(β_saved + s) = η̂` and reproduce the fitted `q` exactly.
2573    // The solver coefficient is already the standard I-spline coefficient:
2574    // observation-space residualization changed the design, not its coefficient
2575    // chart. The family imposes β_w ≥ 0 during the continuously optimized
2576    // constrained REML/LAML fit. Since B' is an M-spline basis with non-negative
2577    // values, dq/dη = 1 + B'(η)·β_w ≥ 1 for every η, including between
2578    // knots; no post-fit sampling or smoothing-parameter ladder is needed.
2579    let saved_warp_beta = fit
2580        .block_states
2581        .get(BinomialMeanWiggleFamily::BLOCK_WIGGLE)
2582        .map(|state| state.beta.to_vec())
2583        .ok_or_else(|| {
2584            "fit_binomial_mean_wiggle: converged fit is missing its LinkWiggle block state"
2585                .to_string()
2586        })?;
2587    validate_monotone_wiggle_beta_nonnegative(
2588        &saved_warp_beta,
2589        "fit_binomial_mean_wiggle saved warp",
2590    )?;
2591    finalize_binomial_mean_wiggle_saved_frame(
2592        &mut fit,
2593        &last_alias,
2594        &x_dense,
2595        &eta_block_input.offset,
2596    )?;
2597    // The frozen-index shift `s = β_frozen_source − β_saved` for the predict
2598    // runtime (#2141). `β_saved` is the just-de-aliased mean block; adding
2599    // `X·s` to the predict base predictor recovers the frozen warp index `η̂`.
2600    // Only meaningful when a warp actually engaged (`saved_warp_beta` present).
2601    let saved_mean_state = fit
2602        .block_states
2603        .get(BinomialMeanWiggleFamily::BLOCK_ETA)
2604        .ok_or_else(|| {
2605            "fit_binomial_mean_wiggle: finalized fit is missing its Mean block state".to_string()
2606        })?;
2607    if frozen_source_beta.len() != saved_mean_state.beta.len() {
2608        return Err(format!(
2609            "fit_binomial_mean_wiggle: frozen-index source has {} coefficients, but saved Mean block has {}",
2610            frozen_source_beta.len(),
2611            saved_mean_state.beta.len(),
2612        ));
2613    }
2614    let saved_index_shift = Some((&frozen_source_beta - &saved_mean_state.beta).to_vec());
2615    Ok(BinomialMeanWiggleFrozenFit {
2616        fit,
2617        saved_warp_beta: Some(saved_warp_beta),
2618        saved_index_shift,
2619        frozen_warp_design,
2620    })
2621}
2622
2623/// Densify a wiggle-block penalty spec to its full `p×p` matrix for the
2624/// observation-space de-aliasing path (#1596). The link-warp block carries only
2625/// `Dense`/`DenseWithMean` difference (and optional ridge) penalties.
2626fn penalty_spec_to_dense(
2627    spec: &crate::model_types::PenaltySpec,
2628    p: usize,
2629) -> Result<Array2<f64>, String> {
2630    use crate::model_types::PenaltySpec;
2631    match spec {
2632        PenaltySpec::Dense(m) | PenaltySpec::DenseWithMean { matrix: m, .. } => {
2633            if m.nrows() != p || m.ncols() != p {
2634                return Err(format!(
2635                    "frozen-basis warp penalty must be {p}x{p}, got {}x{}",
2636                    m.nrows(),
2637                    m.ncols()
2638                ));
2639            }
2640            Ok(m.clone())
2641        }
2642        PenaltySpec::Block {
2643            local, col_range, ..
2644        } => {
2645            let mut full = Array2::<f64>::zeros((p, p));
2646            if col_range.end > p || local.nrows() != col_range.len() {
2647                return Err("frozen-basis warp penalty block range out of bounds".to_string());
2648            }
2649            full.slice_mut(s![col_range.clone(), col_range.clone()])
2650                .assign(local);
2651            Ok(full)
2652        }
2653    }
2654}
2655
2656pub(crate) trait LocationScaleFamilyBuilder {
2657    type Family: CustomFamily + Clone + Send + Sync + 'static;
2658
2659    fn meanspec(&self) -> &TermCollectionSpec;
2660    fn noisespec(&self) -> &TermCollectionSpec;
2661
2662    fn build_blocks(
2663        &self,
2664        theta: &Array1<f64>,
2665        mean_design: &TermCollectionDesign,
2666        noise_design: &TermCollectionDesign,
2667        mean_beta_hint: Option<Array1<f64>>,
2668        noise_beta_hint: Option<Array1<f64>>,
2669    ) -> Result<Vec<ParameterBlockSpec>, String>;
2670
2671    fn build_family(
2672        &self,
2673        mean_design: &TermCollectionDesign,
2674        noise_design: &TermCollectionDesign,
2675    ) -> Self::Family;
2676
2677    fn extract_primary_betas(
2678        &self,
2679        fit: &UnifiedFitResult,
2680    ) -> Result<(Array1<f64>, Array1<f64>), String>;
2681
2682    fn mean_penalty_count(&self, mean_design: &TermCollectionDesign) -> usize {
2683        mean_design.penalties.len()
2684    }
2685
2686    fn noise_penalty_count(&self, noise_design: &TermCollectionDesign) -> usize {
2687        noise_design.penalties.len()
2688    }
2689
2690    fn exact_spatial_joint_supported(&self) -> bool {
2691        false
2692    }
2693
2694    fn require_exact_spatial_joint(&self) -> bool {
2695        false
2696    }
2697
2698    fn exact_spatial_seed_risk_profile(&self) -> crate::seeding::SeedRiskProfile {
2699        crate::seeding::SeedRiskProfile::GeneralizedLinear
2700    }
2701
2702    fn extra_rho0(&self) -> Result<Array1<f64>, String> {
2703        Ok(Array1::zeros(0))
2704    }
2705
2706    fn build_psiderivative_blocks(
2707        &self,
2708        arr: ndarray::ArrayView2<'_, f64>,
2709        term_spec: &TermCollectionSpec,
2710        term_spec2: &TermCollectionSpec,
2711        term_design: &TermCollectionDesign,
2712        term_design2: &TermCollectionDesign,
2713    ) -> Result<Vec<Vec<CustomFamilyBlockPsiDerivative>>, String>;
2714}
2715
2716pub(crate) fn fit_location_scale_terms<B: LocationScaleFamilyBuilder>(
2717    data: ndarray::ArrayView2<'_, f64>,
2718    builder: B,
2719    options: &BlockwiseFitOptions,
2720    kappa_options: &SpatialLengthScaleOptimizationOptions,
2721) -> Result<BlockwiseTermFitResult, String> {
2722    // Large-n location-scale fits keep the caller's explicit Hessian request.
2723    // The unified REML evaluator chooses a dense or matrix-free exact
2724    // representation from the realized (n, p, K) work model, so there is no
2725    // large-scale downgrade to BFGS here.
2726
2727    let mut mean_beta_hint: Option<Array1<f64>> = None;
2728    let mut noise_beta_hint: Option<Array1<f64>> = None;
2729    let extra_rho0 = builder.extra_rho0()?;
2730
2731    let mean_boot_design =
2732        build_term_collection_design(data, builder.meanspec()).map_err(|e| e.to_string())?;
2733    let noise_boot_design =
2734        build_term_collection_design(data, builder.noisespec()).map_err(|e| e.to_string())?;
2735    let mean_bootspec = freeze_term_collection_from_design(builder.meanspec(), &mean_boot_design)
2736        .map_err(|e| e.to_string())?;
2737    let noise_bootspec =
2738        freeze_term_collection_from_design(builder.noisespec(), &noise_boot_design)
2739            .map_err(|e| e.to_string())?;
2740
2741    let require_exact_spatial_joint = builder.require_exact_spatial_joint();
2742    let analytic_joint_derivatives_check = if builder.exact_spatial_joint_supported() {
2743        builder
2744            .build_psiderivative_blocks(
2745                data,
2746                &mean_bootspec,
2747                &noise_bootspec,
2748                &mean_boot_design,
2749                &noise_boot_design,
2750            )
2751            .and_then(|blocks| {
2752                if blocks.is_empty() {
2753                    Err("analytic psi derivative construction produced no parameter blocks"
2754                        .to_string())
2755                } else {
2756                    Ok(())
2757                }
2758            })
2759    } else {
2760        Err(
2761            "analytic spatial psi derivatives are unavailable for this location-scale family"
2762                .to_string(),
2763        )
2764    };
2765    let analytic_joint_derivatives_available = analytic_joint_derivatives_check.is_ok();
2766    if require_exact_spatial_joint {
2767        analytic_joint_derivatives_check.map_err(|err| {
2768            format!("exact two-block spatial path requires analytic psi derivatives: {err}")
2769        })?;
2770    }
2771    let mean_penalty_count = builder.mean_penalty_count(&mean_boot_design);
2772    let noise_penalty_count = builder.noise_penalty_count(&noise_boot_design);
2773
2774    // Honor an explicit user-supplied `length_scale=X` on every spatial term
2775    // in both the mean and noise blocks: when every term is κ-locked (no
2776    // anisotropy, no per-axis ψ contrasts), the joint-spatial outer optimizer
2777    // has nothing to optimize. Routing through it anyway wraps the full
2778    // two-block coefficient solve inside an unnecessary outer loop where
2779    // each evaluation runs the inner Newton from scratch. This is the same
2780    // short-circuit the Bernoulli marginal-slope entry point performs at
2781    // bernoulli_marginal_slope.rs:16432-16442; mirroring it here makes the
2782    // GAMLSS path skip straight to the `(!enabled || log_kappa_dim == 0)`
2783    // fast path in `optimize_spatial_length_scale_exact_joint`.
2784    let mut effective_kappa_options = kappa_options.clone();
2785    if effective_kappa_options.enabled
2786        && gam_terms::smooth::all_spatial_terms_kappa_fixed(&mean_bootspec)
2787        && gam_terms::smooth::all_spatial_terms_kappa_fixed(&noise_bootspec)
2788    {
2789        log::info!(
2790            "[GAMLSS spatial] disabling κ/ψ optimization: every spatial term in \
2791             both blocks has an explicit length_scale and no anisotropy; \
2792             user-supplied kernel scale is fixed"
2793        );
2794        effective_kappa_options.enabled = false;
2795    }
2796    let kappa_options: &SpatialLengthScaleOptimizationOptions = &effective_kappa_options;
2797
2798    // Macro to invoke the exact-joint spatial optimizer with shared closures.
2799    // The exact path evaluates the full profiled/Laplace objective over
2800    // theta = [rho, psi] with the real joint Hessian required by NewtonTR/ARC.
2801    macro_rules! run_exact_joint_spatial {
2802        () => {{
2803            let joint_setup = build_two_block_exact_joint_setup(
2804                data,
2805                builder.meanspec(),
2806                builder.noisespec(),
2807                mean_penalty_count,
2808                noise_penalty_count,
2809                extra_rho0.as_slice().unwrap_or(&[]),
2810                None,
2811                kappa_options,
2812            )
2813            .map_err(|error| error.to_string())?;
2814            let mean_terms = spatial_length_scale_term_indices(builder.meanspec());
2815            let noise_terms = spatial_length_scale_term_indices(builder.noisespec());
2816            let mean_beta_hint_cell = std::cell::RefCell::new(mean_beta_hint.clone());
2817            let noise_beta_hint_cell = std::cell::RefCell::new(noise_beta_hint.clone());
2818            let hyper_warm_start_cell =
2819                std::cell::RefCell::new(None::<CustomFamilyWarmStart>);
2820            // Two-block GAMLSS/location-scale joint likelihoods have a
2821            // β-dependent cross-block Hessian (the (μ,log σ) / (t,log σ)
2822            // off-diagonal blocks involve residual/response scalars that
2823            // shift when β moves). The Wood-Fasiolo structural property
2824            // `H^{-1/2} B_k H^{-1/2} ≽ 0` plus parameter-independent
2825            // nullspace — the mathematical basis for EFS convergence —
2826            // fails here, so EFS/HybridEFS must be excluded at plan time
2827            // rather than retried as a silent first attempt that stalls
2828            // for hundreds of seconds before the runner falls back.
2829            let gamlss_disable_fixed_point = true;
2830            let outer_policy = {
2831                // GAMLSS spatial path: psi_dim = log_kappa_dim + auxiliary_dim,
2832                // matching the (theta_dim - rho_dim) decomposition the
2833                // optimizer uses internally. Build realized ParameterBlockSpecs
2834                // at the seed rho so the family's own cost model — which
2835                // multiplies coefficient-gradient / coefficient-Hessian
2836                // per-row cost by the joint outer-coordinate dimension and
2837                // total p — produces honest `predicted_*_work` estimates.
2838                // Previously this fed `predicted_*_work: 0` to the planner,
2839                // which then ungated dense outer Hessian work that costs
2840                // hundreds of seconds per eval at large scale (see
2841                // `OuterDerivativePolicy::OUTER_HESSIAN_WORK_BUDGET`).
2842                let theta_seed = joint_setup.theta0();
2843                let rho_dim = joint_setup.rho_dim();
2844                let psi_dim = theta_seed.len() - rho_dim;
2845                let rho_seed = theta_seed.slice(s![..rho_dim]).to_owned();
2846                let policy_blocks_res = builder.build_blocks(
2847                    &rho_seed,
2848                    &mean_boot_design,
2849                    &noise_boot_design,
2850                    mean_beta_hint_cell.borrow().clone(),
2851                    noise_beta_hint_cell.borrow().clone(),
2852                );
2853                let mut policy = match policy_blocks_res {
2854                    Ok(policy_blocks) => {
2855                        let policy_family =
2856                            builder.build_family(&mean_boot_design, &noise_boot_design);
2857                        crate::custom_family::CustomFamily::outer_derivative_policy(
2858                            &policy_family,
2859                            &policy_blocks,
2860                            psi_dim,
2861                            options,
2862                        )
2863                    }
2864                    Err(err) => {
2865                        // Block construction at the seed should not fail for
2866                        // any in-tree family, but if it does, fall back to a
2867                        // policy that names the capability honestly and
2868                        // declines to predict cost. Setting work to
2869                        // `u128::MAX` routes the planner through gradient-only
2870                        // BFGS (the universal Hessian-work budget is
2871                        // saturating, so a sentinel is fine here).
2872                        log::warn!(
2873                            "[GAMLSS spatial] failed to realize policy blocks at seed rho ({err}); \
2874                             routing outer optimizer through gradient-only BFGS"
2875                        );
2876                        let capability = if analytic_joint_derivatives_available {
2877                            crate::custom_family::ExactOuterDerivativeOrder::Second
2878                        } else {
2879                            crate::custom_family::ExactOuterDerivativeOrder::First
2880                        };
2881                        crate::custom_family::OuterDerivativePolicy {
2882                            capability,
2883                            predicted_gradient_work: u128::MAX,
2884                            predicted_hessian_work: u128::MAX,
2885                            // No GAMLSS family today overrides its
2886                            // outer-only `_with_options` hooks to consume
2887                            // `outer_score_subsample`; staged-κ would
2888                            // build pilot masks the family then ignores.
2889                            subsample_capable: false,
2890                        }
2891                    }
2892                };
2893                if !analytic_joint_derivatives_available {
2894                    // Capability must not exceed what the analytic derivatives
2895                    // path can supply — the macro's hyper evaluator returns
2896                    // an error otherwise.
2897                    policy.capability =
2898                        crate::custom_family::ExactOuterDerivativeOrder::First;
2899                }
2900                policy
2901            };
2902            optimize_spatial_length_scale_exact_joint(
2903                data,
2904                &[builder.meanspec().clone(), builder.noisespec().clone()],
2905                &[mean_terms, noise_terms],
2906                kappa_options,
2907                &joint_setup,
2908                builder.exact_spatial_seed_risk_profile(),
2909                analytic_joint_derivatives_available,
2910                analytic_joint_derivatives_available,
2911                gamlss_disable_fixed_point,
2912                None,
2913                outer_policy,
2914                |theta,
2915                 specs: &[TermCollectionSpec],
2916                 designs: &[TermCollectionDesign],
2917                 provenance| {
2918                    assert_eq!(
2919                        specs.len(),
2920                        2,
2921                        "joint spatial closure expects exactly two block specs (mean, noise); got {}",
2922                        specs.len(),
2923                    );
2924                    assert_eq!(
2925                        designs.len(),
2926                        2,
2927                        "joint spatial closure expects exactly two block designs (mean, noise); got {}",
2928                        designs.len(),
2929                    );
2930                    let rho = theta.slice(s![..joint_setup.rho_dim()]).to_owned();
2931                    let fit = {
2932                        let blocks = builder.build_blocks(
2933                            &rho,
2934                            &designs[0],
2935                            &designs[1],
2936                            mean_beta_hint_cell.borrow().clone(),
2937                            noise_beta_hint_cell.borrow().clone(),
2938                        )?;
2939                        if mean_beta_hint_cell.borrow().is_none()
2940                            && let Some(beta) = blocks.first().and_then(|block| block.initial_beta.clone())
2941                        {
2942                            *mean_beta_hint_cell.borrow_mut() = Some(beta);
2943                        }
2944                        if noise_beta_hint_cell.borrow().is_none()
2945                            && let Some(beta) =
2946                                blocks.get(1).and_then(|block| block.initial_beta.clone())
2947                        {
2948                            *noise_beta_hint_cell.borrow_mut() = Some(beta);
2949                        }
2950                        let family = builder.build_family(&designs[0], &designs[1]);
2951                        // Branch on whether the κ optimizer drives rho.
2952                        //
2953                        // * `log_kappa_dim() > 0 && kappa_options.enabled` ⇒
2954                        //   the outer (ρ, ψ) optimizer is active and
2955                        //   passes each candidate ρ to this closure;
2956                        //   the inner fit must hold log-lambdas fixed
2957                        //   at the supplied ρ so the outer derivative
2958                        //   has a well-defined directional gradient.
2959                        //
2960                        // * Otherwise (κ disabled via the locked-κ
2961                        //   short-circuit, or no spatial terms at all)
2962                        //   the fast path in
2963                        //   `optimize_spatial_length_scale_exact_joint`
2964                        //   calls this closure exactly once at
2965                        //   `theta = theta0`; ρ must still be optimized
2966                        //   from data because the user never pinned it.
2967                        //   `fit_custom_family` performs the joint
2968                        //   ρ + coefficient REML fit at the user's
2969                        //   (now-fixed) kernel scale, which is the
2970                        //   intended behaviour when `length_scale=…` is
2971                        //   set on every spatial term.
2972                        if joint_setup.log_kappa_dim() > 0 && kappa_options.enabled {
2973                            let (certified_outer, mode) = match provenance {
2974                                SpatialFitProvenance::Certified { outer, mode } => (outer, mode),
2975                                SpatialFitProvenance::NoOuterOptimization => {
2976                                    return Err(
2977                                        "active GAMLSS spatial optimization returned no certified outer provenance"
2978                                            .to_string(),
2979                                    );
2980                                }
2981                            };
2982                            let exact_options =
2983                                crate::outer_subsample::exact_outer_options_for_row_set(
2984                                    options,
2985                                    &crate::row_kernel::RowSet::All,
2986                                );
2987                            fit_custom_family_fixed_log_lambdas_from_owned_mode(
2988                                &family,
2989                                &blocks,
2990                                &exact_options,
2991                                mode,
2992                                theta,
2993                                certified_outer,
2994                            ).map_err(|error| error.to_string())?
2995                        } else {
2996                            fit_custom_family(&family, &blocks, options).map_err(|error| error.to_string())?
2997                        }
2998                    };
2999                    let (mean_beta, noise_beta) = builder.extract_primary_betas(&fit)?;
3000                    mean_beta_hint = Some(mean_beta);
3001                    noise_beta_hint = Some(noise_beta);
3002                    *mean_beta_hint_cell.borrow_mut() = mean_beta_hint.clone();
3003                    *noise_beta_hint_cell.borrow_mut() = noise_beta_hint.clone();
3004                    Ok(fit)
3005                },
3006                |theta,
3007                 specs: &[TermCollectionSpec],
3008                 designs: &[TermCollectionDesign],
3009                 eval_mode,
3010                 row_set: &crate::row_kernel::RowSet,
3011                 _| {
3012                    use gam_problem::EvalMode;
3013                    if !analytic_joint_derivatives_available {
3014                        return Err(
3015                            "analytic spatial psi derivatives are unavailable for this exact two-block path"
3016                                .to_string(),
3017                        );
3018                    }
3019                    let rho = theta.slice(s![..joint_setup.rho_dim()]).to_owned();
3020                    let blocks = builder.build_blocks(
3021                        &rho,
3022                        &designs[0],
3023                        &designs[1],
3024                        mean_beta_hint_cell.borrow().clone(),
3025                        noise_beta_hint_cell.borrow().clone(),
3026                    )?;
3027                    if mean_beta_hint_cell.borrow().is_none()
3028                        && let Some(beta) = blocks.first().and_then(|block| block.initial_beta.clone())
3029                    {
3030                        *mean_beta_hint_cell.borrow_mut() = Some(beta);
3031                    }
3032                    if noise_beta_hint_cell.borrow().is_none()
3033                        && let Some(beta) = blocks.get(1).and_then(|block| block.initial_beta.clone())
3034                    {
3035                        *noise_beta_hint_cell.borrow_mut() = Some(beta);
3036                    }
3037                    let family = builder.build_family(&designs[0], &designs[1]);
3038                    let psiderivative_blocks = builder.build_psiderivative_blocks(
3039                        data,
3040                        &specs[0],
3041                        &specs[1],
3042                        &designs[0],
3043                        &designs[1],
3044                    )?;
3045                    let hyper_layout = crate::custom_family::CustomFamilyHyperLayout::new(
3046                        psiderivative_blocks,
3047                        Vec::new(),
3048                        theta.slice(s![joint_setup.rho_dim()..]).to_owned(),
3049                    )?;
3050                    let warm_start = hyper_warm_start_cell.borrow().clone();
3051                    // Forward the κ-staging row set to the family by installing it
3052                    // on the canonical `outer_score_subsample` option. Inner-PIRLS
3053                    // and final covariance still run on full data (the per-row
3054                    // weight is consulted only by outer-only paths inside the
3055                    // family). When the staging schedule is full-data the option
3056                    // stays `None` and the call is equivalent to the prior path.
3057                    let eval_options =
3058                        crate::outer_subsample::exact_outer_options_for_row_set(options, row_set);
3059                    let owned = evaluate_custom_family_joint_hyper_owned(
3060                        &family,
3061                        &blocks,
3062                        &eval_options,
3063                        &rho,
3064                        &hyper_layout,
3065                        warm_start.as_ref(),
3066                        eval_mode,
3067                    ).map_err(|error| error.to_string())?;
3068                    *hyper_warm_start_cell.borrow_mut() = Some(owned.result.warm_start.clone());
3069                    if !owned.result.inner_converged {
3070                        return Err(
3071                            "exact two-block spatial inner solve did not converge".to_string(),
3072                        );
3073                    }
3074                    if matches!(eval_mode, EvalMode::ValueGradientHessian)
3075                        && !owned.result.outer_hessian.is_analytic()
3076                    {
3077                        return Err(
3078                            "exact two-block spatial objective requires a full joint [rho, psi] hessian"
3079                            .to_string(),
3080                        );
3081                    }
3082                    Ok(ExactJointEvaluation {
3083                        objective: owned.result.objective,
3084                        gradient: owned.result.gradient,
3085                        hessian: owned.result.outer_hessian,
3086                        mode: owned.mode,
3087                    })
3088                },
3089                |theta,
3090                 specs: &[TermCollectionSpec],
3091                 designs: &[TermCollectionDesign],
3092                 row_set: &crate::row_kernel::RowSet| {
3093                    if !analytic_joint_derivatives_available {
3094                        return Err(
3095                            "analytic spatial psi derivatives are unavailable for this exact two-block path"
3096                                .to_string(),
3097                        );
3098                    }
3099                    let rho = theta.slice(s![..joint_setup.rho_dim()]).to_owned();
3100                    let blocks = builder.build_blocks(
3101                        &rho,
3102                        &designs[0],
3103                        &designs[1],
3104                        mean_beta_hint_cell.borrow().clone(),
3105                        noise_beta_hint_cell.borrow().clone(),
3106                    )?;
3107                    if mean_beta_hint_cell.borrow().is_none()
3108                        && let Some(beta) = blocks.first().and_then(|block| block.initial_beta.clone())
3109                    {
3110                        *mean_beta_hint_cell.borrow_mut() = Some(beta);
3111                    }
3112                    if noise_beta_hint_cell.borrow().is_none()
3113                        && let Some(beta) = blocks.get(1).and_then(|block| block.initial_beta.clone())
3114                    {
3115                        *noise_beta_hint_cell.borrow_mut() = Some(beta);
3116                    }
3117                    let family = builder.build_family(&designs[0], &designs[1]);
3118                    let psiderivative_blocks = builder.build_psiderivative_blocks(
3119                        data,
3120                        &specs[0],
3121                        &specs[1],
3122                        &designs[0],
3123                        &designs[1],
3124                    )?;
3125                    let hyper_layout = crate::custom_family::CustomFamilyHyperLayout::new(
3126                        psiderivative_blocks,
3127                        Vec::new(),
3128                        theta.slice(s![joint_setup.rho_dim()..]).to_owned(),
3129                    )?;
3130                    let warm_start = hyper_warm_start_cell.borrow().clone();
3131                    let eval_options =
3132                        crate::outer_subsample::exact_outer_options_for_row_set(options, row_set);
3133                    let owned = evaluate_custom_family_joint_hyper_efs_owned(
3134                        &family,
3135                        &blocks,
3136                        &eval_options,
3137                        &rho,
3138                        &hyper_layout,
3139                        warm_start.as_ref(),
3140                    ).map_err(|error| error.to_string())?;
3141                    *hyper_warm_start_cell.borrow_mut() = Some(owned.result.warm_start.clone());
3142                    if !owned.result.inner_converged {
3143                        return Err(
3144                            "exact two-block spatial EFS inner solve did not converge".to_string(),
3145                        );
3146                    }
3147                    Ok(ExactJointEfsEvaluation {
3148                        evaluation: owned.result.efs_eval,
3149                        mode: owned.mode,
3150                    })
3151                },
3152                |_: &Array1<f64>| Ok(gam_solve::rho_optimizer::SeedOutcome::NoSlot),
3153            )
3154        }};
3155    }
3156
3157    let mut solved = run_exact_joint_spatial!()
3158        .map_err(|err| format!("exact two-block spatial optimization failed: {err}"))?;
3159
3160    let expected_noise_penalty_count = builder.noise_penalty_count(&solved.designs[1]);
3161    let actual_noise_penalty_count = solved.designs[1].penalties.len();
3162    if expected_noise_penalty_count > actual_noise_penalty_count {
3163        if expected_noise_penalty_count != actual_noise_penalty_count + 1 {
3164            return Err(GamlssError::UnsupportedConfiguration {
3165                reason: format!(
3166                    "location-scale result noise design expected {} penalties after augmentation, got {} before augmentation",
3167                    expected_noise_penalty_count, actual_noise_penalty_count
3168                ),
3169            }
3170            .into());
3171        }
3172        append_binomial_log_sigma_shrinkage_penalty_design(&mut solved.designs[1]);
3173    }
3174
3175    BlockwiseTermFitResult::try_from_parts(BlockwiseTermFitResultParts {
3176        fit: solved.fit,
3177        meanspec_resolved: solved.resolved_specs.remove(0),
3178        noisespec_resolved: solved.resolved_specs.remove(0),
3179        mean_design: solved.designs.remove(0),
3180        noise_design: solved.designs.remove(0),
3181    })
3182}
3183
3184pub(crate) struct GaussianLocationScaleTermBuilder {
3185    pub(crate) y: Array1<f64>,
3186    pub(crate) weights: Array1<f64>,
3187    pub(crate) meanspec: TermCollectionSpec,
3188    pub(crate) noisespec: TermCollectionSpec,
3189    pub(crate) mean_offset: Array1<f64>,
3190    pub(crate) noise_offset: Array1<f64>,
3191}
3192
3193impl LocationScaleFamilyBuilder for GaussianLocationScaleTermBuilder {
3194    type Family = GaussianLocationScaleFamily;
3195
3196    fn meanspec(&self) -> &TermCollectionSpec {
3197        &self.meanspec
3198    }
3199
3200    fn noisespec(&self) -> &TermCollectionSpec {
3201        &self.noisespec
3202    }
3203
3204    fn exact_spatial_joint_supported(&self) -> bool {
3205        true
3206    }
3207
3208    fn exact_spatial_seed_risk_profile(&self) -> crate::seeding::SeedRiskProfile {
3209        crate::seeding::SeedRiskProfile::GaussianLocationScale
3210    }
3211
3212    fn build_blocks(
3213        &self,
3214        theta: &Array1<f64>,
3215        mean_design: &TermCollectionDesign,
3216        noise_design: &TermCollectionDesign,
3217        mean_beta_hint: Option<Array1<f64>>,
3218        noise_beta_hint: Option<Array1<f64>>,
3219    ) -> Result<Vec<ParameterBlockSpec>, String> {
3220        let layout = GamlssLambdaLayout::two_block(
3221            mean_design.penalties.len(),
3222            self.noise_penalty_count(noise_design),
3223        );
3224        layout.validate_theta_len(theta.len(), "gaussian location-scale")?;
3225        let (meanspec, noisespec) = build_gaussian_mean_and_scale_blocks(
3226            &self.y,
3227            &self.weights,
3228            mean_design,
3229            noise_design,
3230            &self.mean_offset,
3231            &self.noise_offset,
3232            layout.mean_from(theta),
3233            layout.noise_from(theta),
3234            mean_beta_hint,
3235            noise_beta_hint,
3236            "GaussianLocationScale::build_blocks",
3237        )?;
3238        Ok(vec![meanspec, noisespec])
3239    }
3240
3241    fn build_family(
3242        &self,
3243        mean_design: &TermCollectionDesign,
3244        noise_design: &TermCollectionDesign,
3245    ) -> Self::Family {
3246        let preparednoise_design =
3247            prepared_gaussian_log_sigma_design(&mean_design.design, &noise_design.design)
3248                .expect("prepared Gaussian log-sigma design should match block construction");
3249        GaussianLocationScaleFamily {
3250            y: self.y.clone(),
3251            weights: self.weights.clone(),
3252            mu_design: Some(mean_design.design.clone()),
3253            log_sigma_design: Some(preparednoise_design),
3254            policy: gam_runtime::resource::ResourcePolicy::default_library(),
3255            cached_row_scalars: std::sync::RwLock::new(None),
3256        }
3257    }
3258
3259    fn extract_primary_betas(
3260        &self,
3261        fit: &UnifiedFitResult,
3262    ) -> Result<(Array1<f64>, Array1<f64>), String> {
3263        let mean_beta = fit
3264            .block_states
3265            .get(GaussianLocationScaleFamily::BLOCK_MU)
3266            .ok_or_else(|| "missing Gaussian mu block state".to_string())?
3267            .beta
3268            .clone();
3269        let noise_beta = fit
3270            .block_states
3271            .get(GaussianLocationScaleFamily::BLOCK_LOG_SIGMA)
3272            .ok_or_else(|| "missing Gaussian log_sigma block state".to_string())?
3273            .beta
3274            .clone();
3275        Ok((mean_beta, noise_beta))
3276    }
3277
3278    fn build_psiderivative_blocks(
3279        &self,
3280        data: ndarray::ArrayView2<'_, f64>,
3281        meanspec_resolved: &TermCollectionSpec,
3282        noisespec_resolved: &TermCollectionSpec,
3283        mean_design: &TermCollectionDesign,
3284        noise_design: &TermCollectionDesign,
3285    ) -> Result<Vec<Vec<CustomFamilyBlockPsiDerivative>>, String> {
3286        let mean_derivs =
3287            build_block_spatial_psi_derivatives(data, meanspec_resolved, mean_design)?
3288                .ok_or_else(|| "missing Gaussian mean spatial psi derivatives".to_string())?;
3289        let noise_derivs =
3290            build_block_spatial_psi_derivatives(data, noisespec_resolved, noise_design)?
3291                .ok_or_else(|| "missing Gaussian log-sigma spatial psi derivatives".to_string())?;
3292        Ok(vec![mean_derivs, noise_derivs])
3293    }
3294}
3295
3296pub(crate) struct GaussianLocationScaleWiggleTermBuilder {
3297    pub(crate) y: Array1<f64>,
3298    pub(crate) weights: Array1<f64>,
3299    pub(crate) meanspec: TermCollectionSpec,
3300    pub(crate) noisespec: TermCollectionSpec,
3301    pub(crate) mean_offset: Array1<f64>,
3302    pub(crate) noise_offset: Array1<f64>,
3303    pub(crate) wiggle_knots: Array1<f64>,
3304    pub(crate) wiggle_degree: usize,
3305    pub(crate) wiggle_block: ParameterBlockInput,
3306}
3307
3308impl LocationScaleFamilyBuilder for GaussianLocationScaleWiggleTermBuilder {
3309    type Family = GaussianLocationScaleWiggleFamily;
3310
3311    fn meanspec(&self) -> &TermCollectionSpec {
3312        &self.meanspec
3313    }
3314
3315    fn noisespec(&self) -> &TermCollectionSpec {
3316        &self.noisespec
3317    }
3318
3319    fn exact_spatial_joint_supported(&self) -> bool {
3320        true
3321    }
3322
3323    fn exact_spatial_seed_risk_profile(&self) -> crate::seeding::SeedRiskProfile {
3324        crate::seeding::SeedRiskProfile::GaussianLocationScale
3325    }
3326
3327    fn require_exact_spatial_joint(&self) -> bool {
3328        true
3329    }
3330
3331    fn extra_rho0(&self) -> Result<Array1<f64>, String> {
3332        initial_log_lambdas_orzeros(&self.wiggle_block)
3333    }
3334
3335    fn build_blocks(
3336        &self,
3337        theta: &Array1<f64>,
3338        mean_design: &TermCollectionDesign,
3339        noise_design: &TermCollectionDesign,
3340        mean_beta_hint: Option<Array1<f64>>,
3341        noise_beta_hint: Option<Array1<f64>>,
3342    ) -> Result<Vec<ParameterBlockSpec>, String> {
3343        let layout = GamlssLambdaLayout::withwiggle(
3344            mean_design.penalties.len(),
3345            self.noise_penalty_count(noise_design),
3346            self.wiggle_block.penalties.len(),
3347        );
3348        layout.validate_theta_len(theta.len(), "gaussian location-scale wiggle")?;
3349        let (mut meanspec, mut noisespec) = build_gaussian_mean_and_scale_blocks(
3350            &self.y,
3351            &self.weights,
3352            mean_design,
3353            noise_design,
3354            &self.mean_offset,
3355            &self.noise_offset,
3356            layout.mean_from(theta),
3357            layout.noise_from(theta),
3358            mean_beta_hint,
3359            noise_beta_hint,
3360            "GaussianLocationScaleWiggle::build_blocks",
3361        )?;
3362        // Keep the dynamic full-width wiggle basis safe from a canonical-gauge
3363        // column drop: route the shared level/intercept alias onto the
3364        // column-reducible mean and log-sigma blocks by giving them a lower
3365        // gauge priority than the wiggle block's fixed 100 (see the binomial
3366        // wiggle path and `build_location_scale_wiggle_block`).
3367        meanspec.gauge_priority = LINK_WIGGLE_GAUGE_PRIORITY;
3368        noisespec.gauge_priority = LINK_WIGGLE_GAUGE_PRIORITY;
3369        let n_rows = meanspec.design.nrows();
3370        let wigglespec = build_location_scale_wiggle_block(
3371            "wiggle",
3372            self.wiggle_block.design.clone(),
3373            self.wiggle_block.offset.clone(),
3374            wiggle_block_penalty_matrices(&self.wiggle_block),
3375            self.wiggle_block.nullspace_dims.clone(),
3376            layout.wiggle_from(theta),
3377            self.wiggle_block.initial_beta.clone(),
3378            n_rows,
3379        )?;
3380        Ok(vec![meanspec, noisespec, wigglespec])
3381    }
3382
3383    fn build_family(
3384        &self,
3385        mean_design: &TermCollectionDesign,
3386        noise_design: &TermCollectionDesign,
3387    ) -> Self::Family {
3388        let preparednoise_design =
3389            prepared_gaussian_log_sigma_design(&mean_design.design, &noise_design.design).expect(
3390                "prepared Gaussian log-sigma design should match wiggle block construction",
3391            );
3392        GaussianLocationScaleWiggleFamily {
3393            y: self.y.clone(),
3394            weights: self.weights.clone(),
3395            mu_design: Some(mean_design.design.clone()),
3396            log_sigma_design: Some(preparednoise_design),
3397            wiggle_knots: self.wiggle_knots.clone(),
3398            wiggle_degree: self.wiggle_degree,
3399            policy: gam_runtime::resource::ResourcePolicy::default_library(),
3400            cached_row_scalars: std::sync::RwLock::new(None),
3401        }
3402    }
3403
3404    fn extract_primary_betas(
3405        &self,
3406        fit: &UnifiedFitResult,
3407    ) -> Result<(Array1<f64>, Array1<f64>), String> {
3408        let mean_beta = fit
3409            .block_states
3410            .get(GaussianLocationScaleWiggleFamily::BLOCK_MU)
3411            .ok_or_else(|| "missing Gaussian wiggle mu block state".to_string())?
3412            .beta
3413            .clone();
3414        let noise_beta = fit
3415            .block_states
3416            .get(GaussianLocationScaleWiggleFamily::BLOCK_LOG_SIGMA)
3417            .ok_or_else(|| "missing Gaussian wiggle log_sigma block state".to_string())?
3418            .beta
3419            .clone();
3420        Ok((mean_beta, noise_beta))
3421    }
3422
3423    fn build_psiderivative_blocks(
3424        &self,
3425        data: ndarray::ArrayView2<'_, f64>,
3426        meanspec_resolved: &TermCollectionSpec,
3427        noisespec_resolved: &TermCollectionSpec,
3428        mean_design: &TermCollectionDesign,
3429        noise_design: &TermCollectionDesign,
3430    ) -> Result<Vec<Vec<CustomFamilyBlockPsiDerivative>>, String> {
3431        let mean_derivs =
3432            build_block_spatial_psi_derivatives(data, meanspec_resolved, mean_design)?.ok_or_else(
3433                || "missing Gaussian wiggle mean spatial psi derivatives".to_string(),
3434            )?;
3435        let noise_derivs =
3436            build_block_spatial_psi_derivatives(data, noisespec_resolved, noise_design)?
3437                .ok_or_else(|| {
3438                    "missing Gaussian wiggle log-sigma spatial psi derivatives".to_string()
3439                })?;
3440        Ok(vec![mean_derivs, noise_derivs, Vec::new()])
3441    }
3442}
3443
3444pub(crate) struct BinomialLocationScaleTermBuilder {
3445    pub(crate) y: Array1<f64>,
3446    pub(crate) weights: Array1<f64>,
3447    pub(crate) link_kind: InverseLink,
3448    pub(crate) meanspec: TermCollectionSpec,
3449    pub(crate) noisespec: TermCollectionSpec,
3450    pub(crate) mean_offset: Array1<f64>,
3451    pub(crate) noise_offset: Array1<f64>,
3452}
3453
3454impl LocationScaleFamilyBuilder for BinomialLocationScaleTermBuilder {
3455    type Family = BinomialLocationScaleFamily;
3456
3457    fn meanspec(&self) -> &TermCollectionSpec {
3458        &self.meanspec
3459    }
3460
3461    fn noisespec(&self) -> &TermCollectionSpec {
3462        &self.noisespec
3463    }
3464
3465    fn exact_spatial_joint_supported(&self) -> bool {
3466        true
3467    }
3468
3469    fn require_exact_spatial_joint(&self) -> bool {
3470        true
3471    }
3472
3473    fn noise_penalty_count(&self, noise_design: &TermCollectionDesign) -> usize {
3474        noise_design.penalties.len() + 1
3475    }
3476
3477    fn build_blocks(
3478        &self,
3479        theta: &Array1<f64>,
3480        mean_design: &TermCollectionDesign,
3481        noise_design: &TermCollectionDesign,
3482        mean_beta_hint: Option<Array1<f64>>,
3483        noise_beta_hint: Option<Array1<f64>>,
3484    ) -> Result<Vec<ParameterBlockSpec>, String> {
3485        let layout = GamlssLambdaLayout::two_block(
3486            mean_design.penalties.len(),
3487            self.noise_penalty_count(noise_design),
3488        );
3489        layout.validate_theta_len(theta.len(), "binomial location-scale")?;
3490        let (thresholdspec, log_sigmaspec) = build_binomial_threshold_and_scale_blocks(
3491            &self.y,
3492            &self.weights,
3493            &self.link_kind,
3494            mean_design,
3495            noise_design,
3496            &self.mean_offset,
3497            &self.noise_offset,
3498            layout.mean_from(theta),
3499            layout.noise_from(theta),
3500            mean_beta_hint,
3501            noise_beta_hint,
3502            "BinomialLocationScale::build_blocks",
3503        )?;
3504        Ok(vec![thresholdspec, log_sigmaspec])
3505    }
3506
3507    fn build_family(
3508        &self,
3509        mean_design: &TermCollectionDesign,
3510        noise_design: &TermCollectionDesign,
3511    ) -> Self::Family {
3512        let identifiednoise_design =
3513            identified_binomial_log_sigma_design(mean_design, noise_design, &self.weights)
3514                .expect("identified binomial log-sigma design");
3515        BinomialLocationScaleFamily {
3516            y: self.y.clone(),
3517            weights: self.weights.clone(),
3518            link_kind: self.link_kind.clone(),
3519            threshold_design: Some(mean_design.design.clone()),
3520            log_sigma_design: Some(identifiednoise_design),
3521            policy: gam_runtime::resource::ResourcePolicy::default_library(),
3522        }
3523    }
3524
3525    fn extract_primary_betas(
3526        &self,
3527        fit: &UnifiedFitResult,
3528    ) -> Result<(Array1<f64>, Array1<f64>), String> {
3529        let mean_beta = fit
3530            .block_states
3531            .get(BinomialLocationScaleFamily::BLOCK_T)
3532            .ok_or_else(|| "missing Binomial threshold block state".to_string())?
3533            .beta
3534            .clone();
3535        let noise_beta = fit
3536            .block_states
3537            .get(BinomialLocationScaleFamily::BLOCK_LOG_SIGMA)
3538            .ok_or_else(|| "missing Binomial log_sigma block state".to_string())?
3539            .beta
3540            .clone();
3541        Ok((mean_beta, noise_beta))
3542    }
3543
3544    fn build_psiderivative_blocks(
3545        &self,
3546        data: ndarray::ArrayView2<'_, f64>,
3547        meanspec_resolved: &TermCollectionSpec,
3548        noisespec_resolved: &TermCollectionSpec,
3549        mean_design: &TermCollectionDesign,
3550        noise_design: &TermCollectionDesign,
3551    ) -> Result<Vec<Vec<CustomFamilyBlockPsiDerivative>>, String> {
3552        let mean_derivs =
3553            build_block_spatial_psi_derivatives(data, meanspec_resolved, mean_design)?
3554                .ok_or_else(|| "missing threshold spatial psi derivatives".to_string())?;
3555        let noise_derivs =
3556            build_block_spatial_psi_derivatives(data, noisespec_resolved, noise_design)?
3557                .ok_or_else(|| "missing log_sigma spatial psi derivatives".to_string())?;
3558        Ok(vec![mean_derivs, noise_derivs])
3559    }
3560}
3561
3562pub(crate) struct BinomialLocationScaleWiggleTermBuilder {
3563    pub(crate) y: Array1<f64>,
3564    pub(crate) weights: Array1<f64>,
3565    pub(crate) link_kind: InverseLink,
3566    pub(crate) meanspec: TermCollectionSpec,
3567    pub(crate) noisespec: TermCollectionSpec,
3568    pub(crate) mean_offset: Array1<f64>,
3569    pub(crate) noise_offset: Array1<f64>,
3570    pub(crate) wiggle_knots: Array1<f64>,
3571    pub(crate) wiggle_degree: usize,
3572    pub(crate) wiggle_block: ParameterBlockInput,
3573}
3574
3575impl LocationScaleFamilyBuilder for BinomialLocationScaleWiggleTermBuilder {
3576    type Family = BinomialLocationScaleWiggleFamily;
3577
3578    fn meanspec(&self) -> &TermCollectionSpec {
3579        &self.meanspec
3580    }
3581
3582    fn noisespec(&self) -> &TermCollectionSpec {
3583        &self.noisespec
3584    }
3585
3586    fn exact_spatial_joint_supported(&self) -> bool {
3587        true
3588    }
3589
3590    fn require_exact_spatial_joint(&self) -> bool {
3591        true
3592    }
3593
3594    fn extra_rho0(&self) -> Result<Array1<f64>, String> {
3595        initial_log_lambdas_orzeros(&self.wiggle_block)
3596    }
3597
3598    fn noise_penalty_count(&self, noise_design: &TermCollectionDesign) -> usize {
3599        noise_design.penalties.len() + 1
3600    }
3601
3602    fn build_blocks(
3603        &self,
3604        theta: &Array1<f64>,
3605        mean_design: &TermCollectionDesign,
3606        noise_design: &TermCollectionDesign,
3607        mean_beta_hint: Option<Array1<f64>>,
3608        noise_beta_hint: Option<Array1<f64>>,
3609    ) -> Result<Vec<ParameterBlockSpec>, String> {
3610        let layout = GamlssLambdaLayout::withwiggle(
3611            mean_design.penalties.len(),
3612            self.noise_penalty_count(noise_design),
3613            self.wiggle_block.penalties.len(),
3614        );
3615        layout.validate_theta_len(theta.len(), "wiggle location-scale")?;
3616        let (mut thresholdspec, mut log_sigmaspec) = build_binomial_threshold_and_scale_blocks(
3617            &self.y,
3618            &self.weights,
3619            &self.link_kind,
3620            mean_design,
3621            noise_design,
3622            &self.mean_offset,
3623            &self.noise_offset,
3624            layout.mean_from(theta),
3625            layout.noise_from(theta),
3626            mean_beta_hint,
3627            noise_beta_hint,
3628            "BinomialLocationScaleWiggle::build_blocks",
3629        )?;
3630        // The dynamic monotone wiggle basis is regenerated at full raw width
3631        // every inner iteration and asserts `x.ncols() == spec.design.ncols()`
3632        // in `block_geometry`, so it cannot tolerate a canonical-gauge column
3633        // drop. The level/intercept direction the I-spline shares with the
3634        // threshold block must therefore be routed onto the threshold (and the
3635        // log-sigma) block, whose static designs are column-reducible and
3636        // lifted back via the canonical per-block transform `T`. Give both
3637        // non-wiggle blocks a lower gauge priority than the wiggle block (which
3638        // `build_location_scale_wiggle_block` fixes at 100) so the shared-level
3639        // alias drop lands on them and leaves the dynamic wiggle basis full
3640        // width — mirroring the binomial mean-wiggle path.
3641        thresholdspec.gauge_priority = LINK_WIGGLE_GAUGE_PRIORITY;
3642        log_sigmaspec.gauge_priority = LINK_WIGGLE_GAUGE_PRIORITY;
3643        let n_rows = thresholdspec.design.nrows();
3644        let wigglespec = build_location_scale_wiggle_block(
3645            "wiggle",
3646            self.wiggle_block.design.clone(),
3647            self.wiggle_block.offset.clone(),
3648            wiggle_block_penalty_matrices(&self.wiggle_block),
3649            vec![],
3650            layout.wiggle_from(theta),
3651            self.wiggle_block.initial_beta.clone(),
3652            n_rows,
3653        )?;
3654        Ok(vec![thresholdspec, log_sigmaspec, wigglespec])
3655    }
3656
3657    fn build_family(
3658        &self,
3659        mean_design: &TermCollectionDesign,
3660        noise_design: &TermCollectionDesign,
3661    ) -> Self::Family {
3662        let identifiednoise_design =
3663            identified_binomial_log_sigma_design(mean_design, noise_design, &self.weights)
3664                .expect("identified binomial log-sigma design should match block construction");
3665        BinomialLocationScaleWiggleFamily {
3666            y: self.y.clone(),
3667            weights: self.weights.clone(),
3668            link_kind: self.link_kind.clone(),
3669            threshold_design: Some(mean_design.design.clone()),
3670            log_sigma_design: Some(identifiednoise_design),
3671            wiggle_knots: self.wiggle_knots.clone(),
3672            wiggle_degree: self.wiggle_degree,
3673            policy: gam_runtime::resource::ResourcePolicy::default_library(),
3674        }
3675    }
3676
3677    fn extract_primary_betas(
3678        &self,
3679        fit: &UnifiedFitResult,
3680    ) -> Result<(Array1<f64>, Array1<f64>), String> {
3681        let mean_beta = fit
3682            .block_states
3683            .get(BinomialLocationScaleWiggleFamily::BLOCK_T)
3684            .ok_or_else(|| "missing Binomial wiggle threshold block state".to_string())?
3685            .beta
3686            .clone();
3687        let noise_beta = fit
3688            .block_states
3689            .get(BinomialLocationScaleWiggleFamily::BLOCK_LOG_SIGMA)
3690            .ok_or_else(|| "missing Binomial wiggle log_sigma block state".to_string())?
3691            .beta
3692            .clone();
3693        Ok((mean_beta, noise_beta))
3694    }
3695
3696    fn build_psiderivative_blocks(
3697        &self,
3698        data: ndarray::ArrayView2<'_, f64>,
3699        meanspec_resolved: &TermCollectionSpec,
3700        noisespec_resolved: &TermCollectionSpec,
3701        mean_design: &TermCollectionDesign,
3702        noise_design: &TermCollectionDesign,
3703    ) -> Result<Vec<Vec<CustomFamilyBlockPsiDerivative>>, String> {
3704        let mean_derivs =
3705            build_block_spatial_psi_derivatives(data, meanspec_resolved, mean_design)?
3706                .ok_or_else(|| "missing threshold spatial psi derivatives".to_string())?;
3707        let noise_derivs =
3708            build_block_spatial_psi_derivatives(data, noisespec_resolved, noise_design)?
3709                .ok_or_else(|| "missing log_sigma spatial psi derivatives".to_string())?;
3710        // The wiggle block has no direct spatial design matrix of its own in the
3711        // term builder. Spatial psi moves the wiggle family only through the
3712        // realized threshold/log-sigma designs, which in turn perturb q0 and the
3713        // realized wiggle basis B(q0). The exact joint wiggle psi hooks consume
3714        // those threshold/log-sigma derivative payloads and reconstruct the full
3715        // flattened likelihood-side [rho, psi] calculus internally, so the
3716        // wiggle block intentionally contributes no direct CustomFamilyBlockPsiDerivative
3717        // entries here.
3718        Ok(vec![mean_derivs, noise_derivs, Vec::new()])
3719    }
3720}
3721
3722pub(crate) fn fit_gaussian_location_scale_terms(
3723    data: ndarray::ArrayView2<'_, f64>,
3724    spec: GaussianLocationScaleTermSpec,
3725    options: &BlockwiseFitOptions,
3726    kappa_options: &SpatialLengthScaleOptimizationOptions,
3727) -> Result<BlockwiseTermFitResult, String> {
3728    validate_gaussian_location_scale_termspec(data, &spec, "fit_gaussian_location_scale_terms")?;
3729    fit_location_scale_terms(
3730        data,
3731        GaussianLocationScaleTermBuilder {
3732            y: spec.y,
3733            weights: spec.weights,
3734            meanspec: spec.meanspec,
3735            noisespec: spec.log_sigmaspec,
3736            mean_offset: spec.mean_offset,
3737            noise_offset: spec.log_sigma_offset,
3738        },
3739        options,
3740        kappa_options,
3741    )
3742}
3743
3744pub(crate) fn fit_gaussian_location_scalewiggle_terms(
3745    data: ndarray::ArrayView2<'_, f64>,
3746    spec: GaussianLocationScaleWiggleTermSpec,
3747    options: &BlockwiseFitOptions,
3748    kappa_options: &SpatialLengthScaleOptimizationOptions,
3749) -> Result<BlockwiseTermFitResult, String> {
3750    validate_gaussian_location_scalewiggle_termspec(
3751        data,
3752        &spec,
3753        "fit_gaussian_location_scalewiggle_terms",
3754    )?;
3755    fit_location_scale_terms(
3756        data,
3757        GaussianLocationScaleWiggleTermBuilder {
3758            y: spec.y,
3759            weights: spec.weights,
3760            meanspec: spec.meanspec,
3761            noisespec: spec.log_sigmaspec,
3762            mean_offset: spec.mean_offset,
3763            noise_offset: spec.log_sigma_offset,
3764            wiggle_knots: spec.wiggle_knots,
3765            wiggle_degree: spec.wiggle_degree,
3766            wiggle_block: spec.wiggle_block,
3767        },
3768        options,
3769        kappa_options,
3770    )
3771}
3772
3773pub(crate) fn select_gaussian_location_scale_link_wiggle_basis_from_pilot(
3774    pilot: &BlockwiseTermFitResult,
3775    wiggle_cfg: &WiggleBlockConfig,
3776    wiggle_penalty_orders: &[usize],
3777) -> Result<SelectedWiggleBasis, String> {
3778    let q_seed = pilot
3779        .fit
3780        .block_states
3781        .first()
3782        .ok_or_else(|| "pilot Gaussian wiggle fit is missing mean block".to_string())?
3783        .eta
3784        .view();
3785    select_wiggle_basis_from_seed(q_seed, wiggle_cfg, wiggle_penalty_orders)
3786}
3787
3788pub(crate) fn fit_gaussian_location_scale_terms_with_selected_wiggle(
3789    data: ndarray::ArrayView2<'_, f64>,
3790    spec: GaussianLocationScaleTermSpec,
3791    selected_wiggle_basis: SelectedWiggleBasis,
3792    options: &BlockwiseFitOptions,
3793    kappa_options: &SpatialLengthScaleOptimizationOptions,
3794) -> Result<BlockwiseTermWiggleFitResult, String> {
3795    let SelectedWiggleBasis {
3796        knots: wiggle_knots,
3797        degree: wiggle_degree,
3798        block: wiggle_block,
3799        ..
3800    } = selected_wiggle_basis;
3801    let solved = fit_gaussian_location_scalewiggle_terms(
3802        data,
3803        GaussianLocationScaleWiggleTermSpec {
3804            y: spec.y,
3805            weights: spec.weights,
3806            meanspec: spec.meanspec,
3807            log_sigmaspec: spec.log_sigmaspec,
3808            mean_offset: spec.mean_offset,
3809            log_sigma_offset: spec.log_sigma_offset,
3810            wiggle_knots: wiggle_knots.clone(),
3811            wiggle_degree,
3812            wiggle_block,
3813        },
3814        options,
3815        kappa_options,
3816    )?;
3817
3818    BlockwiseTermWiggleFitResult::try_from_parts(BlockwiseTermWiggleFitResultParts {
3819        fit: solved,
3820        wiggle_knots,
3821        wiggle_degree,
3822    })
3823}
3824
3825pub(crate) fn fit_binomial_location_scale_terms(
3826    data: ndarray::ArrayView2<'_, f64>,
3827    spec: BinomialLocationScaleTermSpec,
3828    options: &BlockwiseFitOptions,
3829    kappa_options: &SpatialLengthScaleOptimizationOptions,
3830) -> Result<BlockwiseTermFitResult, String> {
3831    validate_binomial_location_scale_termspec(data, &spec, "fit_binomial_location_scale_terms")?;
3832    fit_location_scale_terms(
3833        data,
3834        BinomialLocationScaleTermBuilder {
3835            y: spec.y,
3836            weights: spec.weights,
3837            link_kind: spec.link_kind,
3838            meanspec: spec.thresholdspec,
3839            noisespec: spec.log_sigmaspec,
3840            mean_offset: spec.threshold_offset,
3841            noise_offset: spec.log_sigma_offset,
3842        },
3843        options,
3844        kappa_options,
3845    )
3846}
3847
3848pub(crate) fn fit_binomial_location_scalewiggle_terms(
3849    data: ndarray::ArrayView2<'_, f64>,
3850    spec: BinomialLocationScaleWiggleTermSpec,
3851    options: &BlockwiseFitOptions,
3852    kappa_options: &SpatialLengthScaleOptimizationOptions,
3853) -> Result<BlockwiseTermFitResult, String> {
3854    validate_binomial_location_scalewiggle_termspec(
3855        data,
3856        &spec,
3857        "fit_binomial_location_scalewiggle_terms",
3858    )?;
3859    fit_location_scale_terms(
3860        data,
3861        BinomialLocationScaleWiggleTermBuilder {
3862            y: spec.y,
3863            weights: spec.weights,
3864            link_kind: spec.link_kind,
3865            meanspec: spec.thresholdspec,
3866            noisespec: spec.log_sigmaspec,
3867            mean_offset: spec.threshold_offset,
3868            noise_offset: spec.log_sigma_offset,
3869            wiggle_knots: spec.wiggle_knots,
3870            wiggle_degree: spec.wiggle_degree,
3871            wiggle_block: spec.wiggle_block,
3872        },
3873        options,
3874        kappa_options,
3875    )
3876}
3877
3878pub(crate) fn select_binomial_location_scale_link_wiggle_basis_from_pilot(
3879    pilot: &BlockwiseTermFitResult,
3880    wiggle_cfg: &WiggleBlockConfig,
3881    wiggle_penalty_orders: &[usize],
3882) -> Result<SelectedWiggleBasis, String> {
3883    let eta_t = pilot
3884        .fit
3885        .block_states
3886        .first()
3887        .ok_or_else(|| "pilot fit is missing threshold block".to_string())?
3888        .eta
3889        .view();
3890    let eta_ls = pilot
3891        .fit
3892        .block_states
3893        .get(1)
3894        .ok_or_else(|| "pilot fit is missing log_sigma block".to_string())?
3895        .eta
3896        .view();
3897    let sigma = eta_ls.mapv(safe_exp);
3898    let q_seed = Array1::from_iter(eta_t.iter().zip(sigma.iter()).map(|(&t, &s)| -t / s));
3899    select_wiggle_basis_from_seed(q_seed.view(), wiggle_cfg, wiggle_penalty_orders)
3900}
3901
3902pub(crate) fn fit_binomial_location_scale_terms_with_selected_wiggle(
3903    data: ndarray::ArrayView2<'_, f64>,
3904    spec: BinomialLocationScaleTermSpec,
3905    selected_wiggle_basis: SelectedWiggleBasis,
3906    options: &BlockwiseFitOptions,
3907    kappa_options: &SpatialLengthScaleOptimizationOptions,
3908) -> Result<BlockwiseTermWiggleFitResult, String> {
3909    let SelectedWiggleBasis {
3910        knots: wiggle_knots,
3911        degree: wiggle_degree,
3912        block: wiggle_block,
3913        ..
3914    } = selected_wiggle_basis;
3915    let solved = fit_binomial_location_scalewiggle_terms(
3916        data,
3917        BinomialLocationScaleWiggleTermSpec {
3918            y: spec.y,
3919            weights: spec.weights,
3920            link_kind: spec.link_kind,
3921            thresholdspec: spec.thresholdspec,
3922            log_sigmaspec: spec.log_sigmaspec,
3923            threshold_offset: spec.threshold_offset,
3924            log_sigma_offset: spec.log_sigma_offset,
3925            wiggle_knots: wiggle_knots.clone(),
3926            wiggle_degree,
3927            wiggle_block,
3928        },
3929        options,
3930        kappa_options,
3931    )?;
3932
3933    BlockwiseTermWiggleFitResult::try_from_parts(BlockwiseTermWiggleFitResultParts {
3934        fit: solved,
3935        wiggle_knots,
3936        wiggle_degree,
3937    })
3938}
3939
3940pub(crate) fn select_binomial_mean_link_wiggle_basis_from_pilot(
3941    pilot_design: &TermCollectionDesign,
3942    pilot_fit: &UnifiedFitResult,
3943    wiggle_cfg: &WiggleBlockConfig,
3944    wiggle_penalty_orders: &[usize],
3945) -> Result<SelectedWiggleBasis, String> {
3946    let q_seed = pilot_design
3947        .apply(pilot_fit.beta.view())
3948        .map_err(|error| error.to_string())?;
3949    select_wiggle_basis_from_seed(q_seed.view(), wiggle_cfg, wiggle_penalty_orders)
3950}
3951
3952pub(crate) fn fit_binomial_mean_wiggle_terms_with_selected_basis(
3953    data: ndarray::ArrayView2<'_, f64>,
3954    pilot_spec: &TermCollectionSpec,
3955    pilot_design: &TermCollectionDesign,
3956    pilot_fit: &UnifiedFitResult,
3957    y: &Array1<f64>,
3958    weights: &Array1<f64>,
3959    link_kind: InverseLink,
3960    selected_wiggle_basis: SelectedWiggleBasis,
3961    options: &BlockwiseFitOptions,
3962    kappa_options: &SpatialLengthScaleOptimizationOptions,
3963) -> Result<BinomialMeanWiggleTermFitResult, String> {
3964    // The joint `[rho, psi]` box rule, shared with the standard spatial route
3965    // (`spatial_optimization.rs` is `include!`d into `drivers`, so it lives at
3966    // the module root).
3967    use crate::fit_orchestration::drivers::{JOINT_RHO_BOUND, joint_rho_search_box};
3968
3969    validate_term_weights(
3970        data,
3971        y.len(),
3972        weights,
3973        "fit_binomial_mean_wiggle_terms_with_selected_basis",
3974    )?;
3975    validate_binomial_response(y, "fit_binomial_mean_wiggle_terms_with_selected_basis")?;
3976
3977    // Large-n binomial mean-wiggle fits keep the caller's explicit Hessian
3978    // request. The unified evaluator chooses the scalable exact representation
3979    // (dense for small work, operator HVP for large work) instead of routing to
3980    // gradient-only BFGS by observation count.
3981
3982    let SelectedWiggleBasis {
3983        knots: wiggle_knots,
3984        degree: wiggle_degree,
3985        block: wiggle_block,
3986        ..
3987    } = selected_wiggle_basis;
3988
3989    let spatial_terms = spatial_length_scale_term_indices(pilot_spec);
3990    if spatial_terms.is_empty() {
3991        let BinomialMeanWiggleFrozenFit {
3992            fit,
3993            saved_warp_beta,
3994            saved_index_shift,
3995            ..
3996        } = fit_binomial_mean_wiggle(
3997            BinomialMeanWiggleSpec {
3998                y: y.clone(),
3999                weights: weights.clone(),
4000                link_kind,
4001                wiggle_knots: wiggle_knots.clone(),
4002                wiggle_degree,
4003                eta_block: ParameterBlockInput {
4004                    design: pilot_design.design.clone(),
4005                    offset: pilot_design.affine_offset.clone(),
4006                    penalties: pilot_design
4007                        .penalties
4008                        .iter()
4009                        .map(crate::model_types::PenaltySpec::from_blockwise_ref)
4010                        .collect(),
4011                    nullspace_dims: vec![],
4012                    initial_log_lambdas: Some(fitted_log_lambdas(
4013                        &pilot_fit.lambdas,
4014                        "binomial mean-wiggle pilot lambda",
4015                    )?),
4016                    initial_beta: Some(pilot_fit.beta.clone()),
4017                },
4018                wiggle_block,
4019            },
4020            options,
4021        )?;
4022        return Ok(BinomialMeanWiggleTermFitResult {
4023            fit,
4024            resolvedspec: pilot_spec.clone(),
4025            design: pilot_design.clone(),
4026            wiggle_knots,
4027            wiggle_degree,
4028            saved_warp_beta,
4029            saved_index_shift,
4030        });
4031    }
4032
4033    let dims_per_term = spatial_dims_per_term(pilot_spec, &spatial_terms);
4034    let log_kappa0 =
4035        SpatialLogKappaCoords::from_length_scales_aniso(pilot_spec, &spatial_terms, kappa_options)
4036            .reseed_from_data(data, pilot_spec, &spatial_terms, kappa_options)
4037            .map_err(|error| error.to_string())?;
4038    let log_kappa_lower = SpatialLogKappaCoords::lower_bounds_aniso_from_data(
4039        data,
4040        pilot_spec,
4041        &spatial_terms,
4042        &dims_per_term,
4043        kappa_options,
4044    )
4045    .map_err(|error| error.to_string())?;
4046    let log_kappa_upper = SpatialLogKappaCoords::upper_bounds_aniso_from_data(
4047        data,
4048        pilot_spec,
4049        &spatial_terms,
4050        &dims_per_term,
4051        kappa_options,
4052    )
4053    .map_err(|error| error.to_string())?;
4054    // Project seed onto bounds; spec.length_scale is a hint, not a constraint.
4055    let log_kappa0 = log_kappa0.clamp_to_bounds(&log_kappa_lower, &log_kappa_upper);
4056
4057    let eta_penalty_count = pilot_design.penalties.len();
4058    let wiggle_penalty_count = initial_log_lambdas_orzeros(&wiggle_block)?.len();
4059    let rho_dim = eta_penalty_count + wiggle_penalty_count;
4060    let baseline_resolvedspec = log_kappa0
4061        .apply_tospec(pilot_spec, &spatial_terms)
4062        .map_err(|e| e.to_string())?;
4063    let baseline_design =
4064        build_term_collection_design(data, &baseline_resolvedspec).map_err(|e| e.to_string())?;
4065    let baseline = fit_binomial_mean_wiggle(
4066        BinomialMeanWiggleSpec {
4067            y: y.clone(),
4068            weights: weights.clone(),
4069            link_kind: link_kind.clone(),
4070            wiggle_knots: wiggle_knots.clone(),
4071            wiggle_degree,
4072            eta_block: ParameterBlockInput {
4073                design: baseline_design.design.clone(),
4074                offset: baseline_design.affine_offset.clone(),
4075                penalties: baseline_design
4076                    .penalties
4077                    .iter()
4078                    .map(crate::model_types::PenaltySpec::from_blockwise_ref)
4079                    .collect(),
4080                nullspace_dims: vec![],
4081                initial_log_lambdas: Some(fitted_log_lambdas(
4082                    &pilot_fit.lambdas,
4083                    "binomial mean-wiggle pilot lambda",
4084                )?),
4085                initial_beta: Some(pilot_fit.beta.clone()),
4086            },
4087            wiggle_block: wiggle_block.clone(),
4088        },
4089        options,
4090    )?;
4091    let baseline_fit = baseline.fit;
4092    let baseline_log_lambdas = fitted_log_lambdas(
4093        &baseline_fit.lambdas,
4094        "binomial mean-wiggle baseline lambda",
4095    )?;
4096    if baseline_log_lambdas.len() != rho_dim {
4097        return Err(GamlssError::DimensionMismatch {
4098            reason: format!(
4099                "baseline binomial mean-wiggle fit returned {} log-lambdas, expected {rho_dim}",
4100                baseline_log_lambdas.len()
4101            ),
4102        }
4103        .into());
4104    }
4105    // THE WARM START MUST BE IN THE FITTING FRAME, NOT THE SAVED FRAME (#2748).
4106    //
4107    // `fit_binomial_mean_wiggle` returns `beta_saved`, which satisfies
4108    // `q = X*beta_saved + B(eta_hat)*beta_w`: the mean coefficient has absorbed
4109    // `-A*beta_w`. Every consumer below seeds a frozen-index construction with
4110    // it, and `fit_binomial_mean_wiggle`'s own entry computes its FIRST frozen
4111    // index as `X*initial_beta + offset`. Seeded with `beta_saved` that index is
4112    // the DE-ALIASED predictor, so the warp basis is built at the one index the
4113    // mean block already spans — and the refit refuses its own warm start with
4114    // "no identifiable warp direction". MEASURED on `papuan_oce4_matern_k6`: the
4115    // outer search certified `theta*` (`|Pg| = 4.445e-3` against `1.968e-2`,
4116    // `hessian_psd=yes`) and the refit at that same `theta*` then died on exactly
4117    // that message.
4118    //
4119    // `beta_frozen_source = beta_saved + saved_index_shift` is the fitting-frame
4120    // coefficient whose predictor IS `eta_hat` (#2141 defines the shift as
4121    // `beta_frozen_source - beta_saved` for precisely this reconstruction). It is
4122    // also the right frame for the outer evaluator, whose wiggle block is
4123    // `B_perp`: there the model is `q = X*beta + B_perp*beta_w`, which is the
4124    // fitting frame and not the saved one.
4125    let baseline_saved_eta_beta = baseline_fit
4126        .block_states
4127        .get(BinomialMeanWiggleFamily::BLOCK_ETA)
4128        .ok_or_else(|| "baseline binomial mean-wiggle fit missing eta block".to_string())?
4129        .beta
4130        .clone();
4131    let baseline_eta_beta = match baseline.saved_index_shift.as_ref() {
4132        Some(shift) if shift.len() == baseline_saved_eta_beta.len() => {
4133            &baseline_saved_eta_beta + &Array1::from(shift.clone())
4134        }
4135        // No warp engaged (`saved_index_shift` is only produced alongside a
4136        // fitted warp), so the two frames coincide.
4137        None => baseline_saved_eta_beta,
4138        Some(shift) => {
4139            return Err(format!(
4140                "baseline binomial mean-wiggle fit reported a {}-coefficient frozen-index                  shift for a {}-coefficient mean block",
4141                shift.len(),
4142                baseline_saved_eta_beta.len(),
4143            ));
4144        }
4145    };
4146    let baseline_wiggle_beta = Some(
4147        baseline_fit
4148            .block_states
4149            .get(BinomialMeanWiggleFamily::BLOCK_WIGGLE)
4150            .ok_or_else(|| "baseline binomial mean-wiggle fit missing wiggle block".to_string())?
4151            .beta
4152            .clone(),
4153    );
4154    // THE OUTER SEARCH MUST OPTIMIZE THE CRITERION THE FIT IT GRADES REALIZES
4155    // (#2748).
4156    //
4157    // This driver used to hand the outer `[rho, psi]` evaluator the RAW warp
4158    // basis with `frozen_warp_design = None`, i.e. the DYNAMIC `B(eta)` rebuilt
4159    // at the moving index — while the baseline fit above and the final refit
4160    // below are both `fit_binomial_mean_wiggle`, whose whole construction is the
4161    // FROZEN, de-aliased `B_perp = B(eta_hat) - X A`. Three things follow, and
4162    // all three were measured on `papuan_oce4_matern_k6`'s `_flexible` cell:
4163    //
4164    // * **The two are not the same function.** At the identical
4165    //   `rho = [-3.996, -8.097, -1.575, -4.136, -2.873, -10.0]` the frozen path
4166    //   certifies in three cycles at `cost = 1.967e3`; the dynamic path reports
4167    //   `cost = 7.957e8` at `|beta|inf = 1.592e5`. The search was choosing
4168    //   `theta*` for a criterion the refit never evaluates.
4169    //
4170    // * **The frozen one is CONVEX and the dynamic one is not.** With `B` frozen,
4171    //   `q = X*beta + B_perp*beta_w` is linear in both blocks, so the joint
4172    //   Hessian is `[X B_perp]' diag(m2) [X B_perp] + S` with `m2 = w*mu*(1-mu) >= 0`
4173    //   — positive semidefinite by construction. The dynamic arm carries the
4174    //   `d2B/deta2` chain terms and is genuinely indefinite: every seed of this
4175    //   cell reported `resolvable_negative_curvature=true` and ground its full
4176    //   1200-cycle budget at `step_inf = 3.4e4` with the residual at `9.9e2`
4177    //   against a `9.9e-4` tolerance.
4178    //
4179    // * **The declared psi-derivative layout is only complete for the frozen
4180    //   arm.** `CustomFamilyHyperLayout::new(vec![eta_derivs, Vec::new()], ..)`
4181    //   states that the wiggle block's design does not move with psi. Frozen, that
4182    //   is exactly true — `B_perp` is a fixed matrix. Dynamic, it is false:
4183    //   `B(eta)` with `eta = X(psi)*beta` carries `dB/dpsi = B'(eta)*(dX/dpsi)*beta`,
4184    //   a direct design dependence nothing declared, so the analytic outer
4185    //   gradient was missing a term.
4186    //
4187    // The index is frozen ONCE, at the baseline fixed point, and not per outer
4188    // proposal. A per-proposal freeze would make `B_perp` a function of psi and
4189    // reintroduce exactly the undeclared psi dependence of the third point; the
4190    // baseline's own converged index is the only psi-independent choice, and it
4191    // is the point the search starts from. The final refit re-freezes iteratively
4192    // from the pilot, so the approximation lives entirely in which nuisance
4193    // profile the search minimises — not in the fit that is minted.
4194    // The criterion the baseline fixed point realized, taken from the fit rather
4195    // than re-derived: `fit_binomial_mean_wiggle` returns the ACCEPTED `B_perp`
4196    // (`BinomialMeanWiggleFrozenFit::frozen_warp_design`). Re-deriving it here
4197    // was the first attempt at this and it was wrong for a reason worth keeping
4198    // written down: the returned fit is in the SAVED frame, so
4199    // `block_states[BLOCK_ETA].eta` is `X*beta_saved = eta_hat - X*A*beta_w` and
4200    // NOT the frozen index. `wiggle_design` at that predictor is the warp basis
4201    // at the de-aliased index, whose residual against the mean block measured
4202    // `<= 1.33e-9` on `papuan_oce4_matern_k6` — the driver refused its own fit
4203    // with "no identifiable warp direction" while the loop that produced it had
4204    // just converged. One matrix, one owner (#2748).
4205    let frozen_warp_basis = baseline.frozen_warp_design;
4206    let theta_dim = rho_dim + log_kappa0.len();
4207    let mut theta0 = Array1::<f64>::zeros(theta_dim);
4208    theta0
4209        .slice_mut(s![0..rho_dim])
4210        .assign(&baseline_log_lambdas);
4211    theta0
4212        .slice_mut(s![rho_dim..theta_dim])
4213        .assign(log_kappa0.as_array());
4214
4215    // The ρ half of the joint box, through the SAME rule the standard joint
4216    // `[ρ, ψ]` spatial route uses (#2454, corrected by #2760): a `±12` PRIOR
4217    // that falls back per coordinate to the engine's own `±RHO_BOUND` the
4218    // moment the incumbent it will be GRADED against is not strictly inside it.
4219    //
4220    // This site used to write a flat local `RHO_BOUND = 12.0` across every ρ
4221    // coordinate while grading the joint minimum against `baseline_fit` — whose
4222    // λ̂ was found by `fit_binomial_mean_wiggle`'s own outer search over the
4223    // engine's `±30`. When that λ̂ lands outside `±12`, `min` is taken over a set
4224    // that does not contain the point it is compared with, the seed is pasted
4225    // onto the wall, and the coordinate is an ACTIVE constraint from iteration
4226    // zero: `joint_rho_search_box`'s own doc states the mechanism and #2760
4227    // measured it. MEASURED here too, on `papuan_oce4_matern_k6` at
4228    // `5f6bddb16`: all six ρ terminate at `11.9971 … 11.99998` against this
4229    // `±12`, `|Pg| = |g| = 2.163` (so the projection removed NOTHING — every
4230    // railed coordinate still has feasible descent), and the search gives up
4231    // after TWO iterations on a cost stall.
4232    let (rho_lower, rho_upper) = joint_rho_search_box(baseline_log_lambdas.view(), JOINT_RHO_BOUND);
4233    let widened: Vec<usize> = (0..rho_dim)
4234        .filter(|&k| rho_lower[k] < -JOINT_RHO_BOUND || rho_upper[k] > JOINT_RHO_BOUND)
4235        .collect();
4236    if !widened.is_empty() {
4237        log::info!(
4238            "[binomial-mean-wiggle] joint rho box fell back to the engine's own \
4239             +/-RHO_BOUND on coordinate(s) {widened:?}: the baseline fit's own lambda-hat is \
4240             not strictly inside the joint +/-{JOINT_RHO_BOUND} prior, so the prior is \
4241             falsified there and the search region becomes the one the graded incumbent was \
4242             found in (gam#2760). seed={:?} box=[{:?}, {:?}]",
4243            baseline_log_lambdas.to_vec(),
4244            rho_lower.to_vec(),
4245            rho_upper.to_vec(),
4246        );
4247    }
4248    let mut lower = Array1::<f64>::zeros(theta_dim);
4249    let mut upper = Array1::<f64>::zeros(theta_dim);
4250    lower.slice_mut(s![0..rho_dim]).assign(&rho_lower);
4251    upper.slice_mut(s![0..rho_dim]).assign(&rho_upper);
4252    lower
4253        .slice_mut(s![rho_dim..theta_dim])
4254        .assign(log_kappa_lower.as_array());
4255    upper
4256        .slice_mut(s![rho_dim..theta_dim])
4257        .assign(log_kappa_upper.as_array());
4258
4259    let pilot_spec_cloned = pilot_spec.clone();
4260    let pilot_beta = baseline_eta_beta;
4261    // The block the outer evaluator solves is the frozen de-aliased warp, so its
4262    // DESIGN is that matrix and not the raw basis (#2748). `block_geometry`'s
4263    // frozen arm serves `frozen_warp_design` and asserts it matches
4264    // `spec.design.ncols()`; `B_perp` has the same width as `B` by construction
4265    // (`B - X A`), so the two agree and the family's own width check is what
4266    // proves it.
4267    let wiggle_design = DesignMatrix::Dense(gam_linalg::matrix::DenseDesignMatrix::from(
4268        frozen_warp_basis.as_ref().clone(),
4269    ));
4270    let wiggle_offset = Array1::<f64>::zeros(frozen_warp_basis.nrows());
4271    let wiggle_penalties: Vec<crate::model_types::PenaltySpec> = wiggle_block
4272        .penalties
4273        .iter()
4274        .map(|penalty| {
4275            penalty_spec_to_dense(penalty, frozen_warp_basis.ncols())
4276                .map(crate::model_types::PenaltySpec::Dense)
4277        })
4278        .collect::<Result<_, String>>()?;
4279    let wiggle_initial_beta = baseline_wiggle_beta;
4280    let wiggle_knots_cloned = wiggle_knots.clone();
4281    let y_cloned = y.clone();
4282    let weights_cloned = weights.clone();
4283    let link_kind_cloned = link_kind.clone();
4284    let outer_family = BinomialMeanWiggleFamily {
4285        y: y_cloned.clone(),
4286        weights: weights_cloned.clone(),
4287        link_kind: link_kind_cloned.clone(),
4288        wiggle_knots: wiggle_knots_cloned.clone(),
4289        wiggle_degree,
4290        policy: gam_runtime::resource::ResourcePolicy::default_library(),
4291        // The FROZEN, de-aliased basis the baseline fixed point realized — the
4292        // same object `fit_binomial_mean_wiggle` fits and the same object the
4293        // refit below re-derives. See `frozen_warp_basis` above for the three
4294        // measured reasons this is not the dynamic `B(eta)` it used to be
4295        // (#2748): the two are different functions, only this one is convex, and
4296        // only this one makes the declared ψ-derivative layout complete.
4297        frozen_warp_design: Some(std::sync::Arc::clone(&frozen_warp_basis)),
4298    };
4299    let screening_cap = Arc::new(AtomicUsize::new(0));
4300    let mut outer_options = options.clone();
4301    outer_options.screening_max_inner_iterations = Some(Arc::clone(&screening_cap));
4302    struct MeanWiggleOuterState {
4303        pub(crate) warm_cache: Option<crate::custom_family::CustomFamilyWarmStart>,
4304        pub(crate) last_eval: Option<(
4305            Array1<f64>,
4306            f64,
4307            Array1<f64>,
4308            gam_problem::HessianValue,
4309            crate::custom_family::CustomFamilyWarmStart,
4310        )>,
4311    }
4312
4313    let build_realized_blocks = |theta: &Array1<f64>| -> Result<
4314        (
4315            TermCollectionSpec,
4316            TermCollectionDesign,
4317            Vec<ParameterBlockSpec>,
4318            Vec<CustomFamilyBlockPsiDerivative>,
4319        ),
4320        String,
4321    > {
4322        let log_kappa =
4323            SpatialLogKappaCoords::from_theta_tail_with_dims(theta, rho_dim, dims_per_term.clone());
4324        let resolvedspec = log_kappa
4325            .apply_tospec(&pilot_spec_cloned, &spatial_terms)
4326            .map_err(|e| e.to_string())?;
4327        let design =
4328            build_term_collection_design(data, &resolvedspec).map_err(|e| e.to_string())?;
4329        let eta_derivs = build_block_spatial_psi_derivatives(data, &resolvedspec, &design)?
4330            .ok_or_else(|| {
4331                "missing eta spatial psi derivatives for binomial mean wiggle".to_string()
4332            })?;
4333        let blocks = vec![
4334            ParameterBlockSpec {
4335                name: "eta".to_string(),
4336                design: design.design.clone(),
4337                offset: design.affine_offset.clone(),
4338                penalties: design.penalties_as_penalty_matrix(),
4339                nullspace_dims: vec![],
4340                initial_log_lambdas: theta.slice(s![0..eta_penalty_count]).to_owned(),
4341                initial_beta: Some(pilot_beta.clone()),
4342                // The warp block here is the RESIDUAL `B_perp`, so precedence
4343                // runs the same way it does in `fit_binomial_mean_wiggle`: a
4344                // direction the two still share after residualization is one the
4345                // mean block can represent and the warp is echoing, so the mean
4346                // block keeps its priority and the warp yields
4347                // ([`DEALIASED_WARP_GAUGE_PRIORITY`]). This used to be the other
4348                // way round because the warp was the DYNAMIC full-width basis,
4349                // residualized against nothing (#2748).
4350                gauge_priority: DEFAULT_GAUGE_PRIORITY,
4351                jacobian_callback: None,
4352                stacked_design: None,
4353                stacked_offset: None,
4354            },
4355            ParameterBlockSpec {
4356                name: "wiggle".to_string(),
4357                design: wiggle_design.clone(),
4358                offset: wiggle_offset.clone(),
4359                penalties: {
4360                    let p_wiggle = wiggle_design.ncols();
4361                    wiggle_penalties
4362                        .iter()
4363                        .map(|spec| match spec {
4364                            crate::model_types::PenaltySpec::Block {
4365                                local, col_range, ..
4366                            } => PenaltyMatrix::Blockwise {
4367                                local: local.clone(),
4368                                col_range: col_range.clone(),
4369                                total_dim: p_wiggle,
4370                            },
4371                            crate::model_types::PenaltySpec::Dense(m)
4372                            | crate::model_types::PenaltySpec::DenseWithMean {
4373                                matrix: m, ..
4374                            } => PenaltyMatrix::Dense(m.clone()),
4375                        })
4376                        .collect()
4377                },
4378                nullspace_dims: vec![],
4379                initial_log_lambdas: theta.slice(s![eta_penalty_count..rho_dim]).to_owned(),
4380                initial_beta: wiggle_initial_beta.clone(),
4381                gauge_priority: DEALIASED_WARP_GAUGE_PRIORITY,
4382                jacobian_callback: None,
4383                stacked_design: None,
4384                stacked_offset: None,
4385            },
4386        ];
4387        Ok((resolvedspec, design, blocks, eta_derivs))
4388    };
4389
4390    let build_eval = |theta: &Array1<f64>,
4391                      warm_cache: Option<&crate::custom_family::CustomFamilyWarmStart>,
4392                      need_hessian: bool|
4393     -> Result<
4394        (
4395            crate::custom_family::CustomFamilyJointHyperResult,
4396            TermCollectionSpec,
4397            TermCollectionDesign,
4398        ),
4399        String,
4400    > {
4401        let (resolvedspec, design, blocks, eta_derivs) = build_realized_blocks(theta)?;
4402        let hyper_layout = crate::custom_family::CustomFamilyHyperLayout::new(
4403            vec![eta_derivs, Vec::new()],
4404            Vec::new(),
4405            theta.slice(s![rho_dim..]).to_owned(),
4406        )?;
4407        let eval = evaluate_custom_family_joint_hyper(
4408            &outer_family,
4409            &blocks,
4410            &outer_options,
4411            &theta.slice(s![0..rho_dim]).to_owned(),
4412            &hyper_layout,
4413            warm_cache,
4414            if need_hessian {
4415                gam_problem::EvalMode::ValueGradientHessian
4416            } else {
4417                gam_problem::EvalMode::ValueAndGradient
4418            },
4419        ).map_err(|error| error.to_string())?;
4420        Ok((eval, resolvedspec, design))
4421    };
4422
4423    let build_efs = |theta: &Array1<f64>,
4424                     warm_cache: Option<&crate::custom_family::CustomFamilyWarmStart>|
4425     -> Result<crate::custom_family::CustomFamilyJointHyperEfsResult, String> {
4426        let (_, _, blocks, eta_derivs) = build_realized_blocks(theta)?;
4427        let hyper_layout = crate::custom_family::CustomFamilyHyperLayout::new(
4428            vec![eta_derivs, Vec::new()],
4429            Vec::new(),
4430            theta.slice(s![rho_dim..]).to_owned(),
4431        )?;
4432        evaluate_custom_family_joint_hyper_efs(
4433            &outer_family,
4434            &blocks,
4435            &outer_options,
4436            &theta.slice(s![0..rho_dim]).to_owned(),
4437            &hyper_layout,
4438            warm_cache,
4439        )
4440        .map_err(|e| e.to_string())
4441    };
4442
4443    use crate::model_types::EstimationError;
4444    use gam_problem::{DeclaredHessianForm, Derivative, OuterEval};
4445    use gam_solve::rho_optimizer::OuterEvalOrder;
4446
4447    // Exact first-order AND second-order [rho, psi] calculus is available
4448    // for all inverse links via the shared jet formulas plus the generic
4449    // exact-Newton D_βH / D²_βH closures routed through
4450    // evaluate_custom_family_joint_hyper -> joint_outer_evaluate ->
4451    // BorrowedJointDerivProvider. Search consumes the exact first-order lane;
4452    // the terminal certificate alone consumes the exact Hessian.
4453    //
4454    // Spatial log-kappa coordinates are ψ (design-moving) dimensions because
4455    // they rebuild the spatial basis and penalties at each outer proposal.
4456    let analytic_outer_hessian_available = true;
4457    let mut seed_heuristic = theta0.to_vec();
4458    for value in &mut seed_heuristic[..rho_dim] {
4459        *value = value.exp();
4460    }
4461    let problem = gam_solve::rho_optimizer::OuterProblem::new(theta_dim)
4462        .with_gradient(Derivative::Analytic)
4463        .with_hessian(if analytic_outer_hessian_available {
4464            DeclaredHessianForm::Either
4465        } else {
4466            DeclaredHessianForm::Unavailable
4467        })
4468        // #2359: the family gradient consumes derivatives through order three;
4469        // reserve its order-four Hessian for the terminal mint certificate.
4470        .with_prefer_gradient_only(true)
4471        .with_psi_dim(theta_dim - rho_dim)
4472        .with_tolerance(options.outer_tol)
4473        .with_max_iter(options.outer_max_iter)
4474        .with_bounds(lower.clone(), upper.clone())
4475        .with_initial_rho(theta0.clone())
4476        .with_seed_config(crate::seeding::SeedConfig {
4477            max_seeds: 4,
4478            seed_budget: 2,
4479            risk_profile: crate::seeding::SeedRiskProfile::GeneralizedLinear,
4480            num_auxiliary_trailing: theta_dim - rho_dim,
4481            ..Default::default()
4482        })
4483        .with_screening_cap(Arc::clone(&screening_cap))
4484        // The saturation REFERENCE stays at the joint prior even where the box
4485        // above widened a coordinate to `±RHO_BOUND`, exactly as the standard
4486        // joint `[ρ, ψ]` route does: the box is the per-dimension `lower`/`upper`
4487        // pair, and this scalar only feeds the seed grid and the bound-free
4488        // fallback box.
4489        .with_rho_bound(JOINT_RHO_BOUND)
4490        .with_heuristic_lambdas(seed_heuristic);
4491
4492    let eval_outer = |state: &mut MeanWiggleOuterState,
4493                      theta: &Array1<f64>,
4494                      order: OuterEvalOrder|
4495     -> Result<OuterEval, EstimationError> {
4496        if let Some((cached_theta, cached_cost, cached_grad, cached_hess, cached_warm)) =
4497            &state.last_eval
4498            && cached_theta == theta
4499            && (!matches!(order, OuterEvalOrder::ValueGradientHessian)
4500                || matches!(
4501                    cached_hess,
4502                    gam_problem::HessianValue::Dense(_) | gam_problem::HessianValue::Operator(_)
4503                ))
4504        {
4505            state.warm_cache = Some(cached_warm.clone());
4506            return Ok(OuterEval {
4507                cost: *cached_cost,
4508                gradient: cached_grad.clone(),
4509                hessian: cached_hess.clone(),
4510                inner_beta_hint: None,
4511            });
4512        }
4513        let need_hessian = matches!(order, OuterEvalOrder::ValueGradientHessian)
4514            && analytic_outer_hessian_available;
4515        // The evaluator's contract is "produce the criterion at THIS theta",
4516        // so its refusals are per-theta refusals. `InvalidInput` says the
4517        // opposite to `is_trial_point_infeasible`, and the outer boundary then
4518        // graded them Fatal and killed the fit (#2627).
4519        let (eval, _, _) = build_eval(theta, state.warm_cache.as_ref(), need_hessian)
4520            .map_err(|reason| EstimationError::TrialPointRefused { reason })?;
4521        if !eval.inner_converged {
4522            state.warm_cache = Some(eval.warm_start);
4523            return Err(EstimationError::TrialPointRefused {
4524                reason: "binomial mean-wiggle exact spatial inner solve did not converge"
4525                    .to_string(),
4526            });
4527        }
4528        let hessian_result = eval.outer_hessian.clone();
4529        state.last_eval = Some((
4530            theta.clone(),
4531            eval.objective,
4532            eval.gradient.clone(),
4533            eval.outer_hessian.clone(),
4534            eval.warm_start.clone(),
4535        ));
4536        state.warm_cache = Some(eval.warm_start);
4537        Ok(OuterEval {
4538            cost: eval.objective,
4539            gradient: eval.gradient,
4540            hessian: hessian_result,
4541            inner_beta_hint: None,
4542        })
4543    };
4544
4545    let mut obj = problem.build_objective_with_screening_proxy(
4546        MeanWiggleOuterState {
4547            warm_cache: None,
4548            last_eval: None,
4549        },
4550        |state: &mut MeanWiggleOuterState, theta: &Array1<f64>| {
4551            if let Some((cached_theta, cached_cost, _, _, cached_warm)) = &state.last_eval
4552                && cached_theta == theta
4553            {
4554                state.warm_cache = Some(cached_warm.clone());
4555                return Ok(*cached_cost);
4556            }
4557            let (eval, _, _) = build_eval(theta, state.warm_cache.as_ref(), false)
4558                .map_err(|reason| EstimationError::TrialPointRefused { reason })?;
4559            if !eval.inner_converged {
4560                state.warm_cache = Some(eval.warm_start);
4561                return Err(EstimationError::TrialPointRefused {
4562                    reason: "binomial mean-wiggle exact spatial cost inner solve did not converge"
4563                        .to_string(),
4564                });
4565            }
4566            state.warm_cache = Some(eval.warm_start);
4567            Ok(eval.objective)
4568        },
4569        |state: &mut MeanWiggleOuterState, theta: &Array1<f64>| {
4570            eval_outer(
4571                state,
4572                theta,
4573                if analytic_outer_hessian_available {
4574                    OuterEvalOrder::ValueGradientHessian
4575                } else {
4576                    OuterEvalOrder::ValueAndGradient
4577                },
4578            )
4579        },
4580        |state: &mut MeanWiggleOuterState, theta: &Array1<f64>, order: OuterEvalOrder| {
4581            eval_outer(state, theta, order)
4582        },
4583        Some(|state: &mut MeanWiggleOuterState| {
4584            state.warm_cache = None;
4585            state.last_eval = None;
4586        }),
4587        Some(|state: &mut MeanWiggleOuterState, theta: &Array1<f64>| {
4588            let eval = build_efs(theta, state.warm_cache.as_ref())
4589                .map_err(|reason| EstimationError::TrialPointRefused { reason })?;
4590            if !eval.inner_converged {
4591                state.warm_cache = Some(eval.warm_start);
4592                return Err(EstimationError::TrialPointRefused {
4593                    reason: "binomial mean-wiggle exact spatial EFS inner solve did not converge"
4594                        .to_string(),
4595                });
4596            }
4597            state.warm_cache = Some(eval.warm_start);
4598            Ok(eval.efs_eval)
4599        }),
4600        // Seed-screening ranking proxy (#969). The cost closure above
4601        // hard-errors on a non-converged inner solve — correct for
4602        // line-search costs, but under the screening cap (wired into the
4603        // outer options and installed by the cascade) the inner solve is
4604        // truncated BY DESIGN, so screening through it rejects every seed
4605        // — the all-seeds-rejected front-door genus. Screening only RANKS
4606        // candidates: the truncated solve's penalized objective is the
4607        // ranking signal; convergence is demanded of the selected seed's
4608        // full-budget fit, not of capped probes.
4609        |state: &mut MeanWiggleOuterState, theta: &Array1<f64>| {
4610            if let Some((cached_theta, cached_cost, _, _, cached_warm)) = &state.last_eval
4611                && cached_theta == theta
4612            {
4613                state.warm_cache = Some(cached_warm.clone());
4614                return Ok(*cached_cost);
4615            }
4616            let (eval, _, _) = build_eval(theta, state.warm_cache.as_ref(), false)
4617                .map_err(|reason| EstimationError::TrialPointRefused { reason })?;
4618            state.warm_cache = Some(eval.warm_start);
4619            Ok(eval.objective)
4620        },
4621    );
4622
4623    let outer = problem
4624        .run(&mut obj, "binomial mean wiggle exact spatial hyper")
4625        .map_err(|e| e.to_string())?;
4626    if !outer.converged() {
4627        return Err(GamlssError::NumericalFailure { reason: format!(
4628            "binomial mean wiggle exact spatial hyper did not converge after {} iterations (final_objective={:.6e}, final_grad_norm={})",
4629            outer.iterations,
4630            outer.final_value,
4631            outer.final_grad_norm_report(),
4632        ) }.into());
4633    }
4634    let theta_star = outer.rho;
4635
4636    let log_kappa =
4637        SpatialLogKappaCoords::from_theta_tail_with_dims(&theta_star, rho_dim, dims_per_term);
4638    let resolvedspec = log_kappa
4639        .apply_tospec(&pilot_spec_cloned, &spatial_terms)
4640        .map_err(|e| e.to_string())?;
4641    let design = build_term_collection_design(data, &resolvedspec).map_err(|e| e.to_string())?;
4642    let resolvedspec =
4643        freeze_term_collection_from_design(&resolvedspec, &design).map_err(|e| e.to_string())?;
4644    let fit = fit_binomial_mean_wiggle(
4645        BinomialMeanWiggleSpec {
4646            y: y_cloned,
4647            weights: weights_cloned,
4648            link_kind: link_kind_cloned,
4649            wiggle_knots: wiggle_knots.clone(),
4650            wiggle_degree,
4651            eta_block: ParameterBlockInput {
4652                design: design.design.clone(),
4653                offset: design.affine_offset.clone(),
4654                penalties: design
4655                    .penalties
4656                    .iter()
4657                    .map(crate::model_types::PenaltySpec::from_blockwise_ref)
4658                    .collect(),
4659                nullspace_dims: vec![],
4660                initial_log_lambdas: Some(theta_star.slice(s![0..eta_penalty_count]).to_owned()),
4661                initial_beta: Some(pilot_beta),
4662            },
4663            wiggle_block: ParameterBlockInput {
4664                design: wiggle_design,
4665                offset: wiggle_offset,
4666                penalties: wiggle_penalties,
4667                nullspace_dims: vec![],
4668                initial_log_lambdas: Some(
4669                    theta_star.slice(s![eta_penalty_count..rho_dim]).to_owned(),
4670                ),
4671                initial_beta: wiggle_initial_beta,
4672            },
4673        },
4674        options,
4675    )?;
4676    let BinomialMeanWiggleFrozenFit {
4677        fit,
4678        saved_warp_beta,
4679        saved_index_shift,
4680        ..
4681    } = fit;
4682
4683    Ok(BinomialMeanWiggleTermFitResult {
4684        fit,
4685        resolvedspec,
4686        design,
4687        wiggle_knots,
4688        wiggle_degree,
4689        saved_warp_beta,
4690        saved_index_shift,
4691    })
4692}