dsfb_semiconductor/

secom_addendum.rs

use crate::baselines::BaselineSet;
use crate::cohort::OptimizationExecution;
use crate::error::{DsfbSemiconductorError, Result};
use crate::failure_driven::FailureDrivenArtifacts;
use crate::grammar::{GrammarSet, GrammarState};
use crate::metrics::BenchmarkMetrics;
use crate::precursor::{DsaEvaluation, DsaPolicyState};
use crate::preprocessing::PreparedDataset;
use crate::residual::ResidualSet;
use crate::semiotics::{DsfbMotifClass, MotifSet, SemanticLayer};
use plotters::prelude::*;
use serde::Serialize;
use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;

const TRADEOFF_PLOT_WIDTH: u32 = 1400;
const TRADEOFF_PLOT_HEIGHT: u32 = 800;

#[derive(Debug, Clone, Serialize)]
pub struct RecurrentBoundaryStats {
    pub total_boundary_points: usize,
    pub total_run_hits: usize,
    pub total_pre_failure_hits: usize,
    pub pass_run_hits: usize,
    pub precision_pre_failure: f64,
    pub precision_pass: f64,
}

#[derive(Debug, Clone, Serialize)]
pub struct RecurrentBoundaryTradeoffRow {
    pub suppression_level: f64,
    pub suppression_label: String,
    pub investigation_points: usize,
    pub pass_run_nuisance_proxy: f64,
    pub delta_nuisance_vs_selected_dsa: f64,
    pub delta_nuisance_vs_ewma: f64,
    pub failure_recall: usize,
    pub failure_runs: usize,
    pub recall_rate: f64,
}

#[derive(Debug, Clone, Serialize)]
pub struct MetricRegroundingRow {
    pub metric: String,
    pub baseline: String,
    pub dsfb_value: f64,
    pub baseline_value: f64,
    pub delta_percent: f64,
}

#[derive(Debug, Clone, Serialize)]
pub struct TargetDRegressionAnalysis {
    pub contributing_features: Vec<String>,
    pub contributing_motifs: Vec<String>,
    pub contributing_heuristics: Vec<String>,
    pub contributing_policy_rules: Vec<String>,
    pub causal_chain: Vec<String>,
    pub why_regression_occurred: String,
    pub action_taken: String,
    pub tradeoff_justification: String,
}

#[derive(Debug, Clone, Serialize)]
pub struct MissedFailureFeatureActivity {
    pub feature_name: String,
    pub behavior_classification: String,
    pub max_dsa_score: f64,
    pub initial_motif_hypothesis: String,
    pub dominant_dsfb_motif: String,
    pub dominant_grammar_state: String,
    pub failure_explanation: String,
}

#[derive(Debug, Clone, Serialize)]
pub struct MissedFailureRootCause {
    pub failure_id: usize,
    pub feature_activity: Vec<MissedFailureFeatureActivity>,
    pub residual_trajectory: Vec<f64>,
    pub drift_trajectory: Vec<f64>,
    pub slew_trajectory: Vec<f64>,
    pub motif_presence: Vec<String>,
    pub grammar_state: Vec<String>,
    pub reason_for_miss: String,
    pub classification: String,
    pub recovered_after_fix: bool,
    pub recovery_feature: Option<String>,
    pub recovery_note: String,
}

#[derive(Debug, Clone, Serialize)]
pub struct LeadTimeComparisonRow {
    pub failure_id: usize,
    pub dsfb_lead_runs: Option<usize>,
    pub threshold_lead_runs: Option<usize>,
    pub earliest_semantic_match_lead_runs: Option<usize>,
    pub threshold_minus_dsfb_runs: Option<i64>,
    pub threshold_minus_semantic_match_runs: Option<i64>,
}

#[derive(Debug, Clone, Serialize)]
pub struct LeadTimeExplanation {
    pub mean_dsfb_lead_runs: Option<f64>,
    pub mean_threshold_lead_runs: Option<f64>,
    pub mean_semantic_match_lead_runs: Option<f64>,
    pub threshold_earlier_failure_count: usize,
    pub dsfb_earlier_failure_count: usize,
    pub semantic_match_precedes_threshold_count: usize,
    pub motif_emergence_precedes_threshold_count: usize,
    pub explanation: String,
    pub validation_note: String,
}

#[derive(Debug, Clone, Serialize)]
pub struct EpisodePrecisionMetrics {
    pub dsfb_episode_count: usize,
    pub dsfb_pre_failure_episode_count: usize,
    pub dsfb_precision: f64,
    pub raw_alarm_count: usize,
    pub raw_alarm_precision: f64,
    pub precision_gain_factor: f64,
}

#[derive(Debug, Clone, Serialize)]
pub struct SecomAddendumArtifacts {
    pub recurrent_boundary_stats: RecurrentBoundaryStats,
    pub recurrent_boundary_tradeoff_curve: Vec<RecurrentBoundaryTradeoffRow>,
    pub required_tradeoff_statement: String,
    pub required_tradeoff_statement_supported: bool,
    pub metric_regrounding: Vec<MetricRegroundingRow>,
    pub target_d_regression_analysis: TargetDRegressionAnalysis,
    pub missed_failure_root_cause: MissedFailureRootCause,
    pub lead_time_comparison: Vec<LeadTimeComparisonRow>,
    pub lead_time_explanation: LeadTimeExplanation,
    pub episode_precision_metrics: EpisodePrecisionMetrics,
    pub executive_summary_text: String,
    pub paper_abstract_artifact: String,
}

pub fn build_secom_addendum_artifacts(
    dataset: &PreparedDataset,
    residuals: &ResidualSet,
    baselines: &BaselineSet,
    grammar: &GrammarSet,
    motifs: &MotifSet,
    semantic_layer: &SemanticLayer,
    metrics: &BenchmarkMetrics,
    optimization: &OptimizationExecution,
    failure_driven: &FailureDrivenArtifacts,
    baseline_dsa: &DsaEvaluation,
    optimized_dsa: &DsaEvaluation,
    pre_failure_lookback_runs: usize,
) -> SecomAddendumArtifacts {
    let recurrent_boundary_stats =
        build_recurrent_boundary_stats(dataset, motifs, pre_failure_lookback_runs);
    let recurrent_boundary_tradeoff_curve = build_recurrent_boundary_tradeoff_curve(
        dataset,
        grammar,
        motifs,
        metrics,
        optimized_dsa,
        pre_failure_lookback_runs,
    );
    let required_tradeoff_statement = "Suppressing recurrent_boundary_approach sufficiently to achieve ≥40% nuisance reduction on SECOM necessarily reduces recall due to shared structural origin.".to_string();
    let required_tradeoff_statement_supported =
        recurrent_boundary_tradeoff_statement_supported(&recurrent_boundary_tradeoff_curve);
    let metric_regrounding = build_metric_regrounding(
        dataset,
        residuals,
        baselines,
        metrics,
        optimized_dsa,
        pre_failure_lookback_runs,
    );
    let target_d_regression_analysis = build_target_d_regression_analysis(
        dataset,
        grammar,
        motifs,
        failure_driven,
        baseline_dsa,
        optimized_dsa,
        &recurrent_boundary_tradeoff_curve,
    );
    let missed_failure_root_cause = build_missed_failure_root_cause(failure_driven);
    let lead_time_comparison = build_lead_time_comparison(
        semantic_layer,
        optimized_dsa,
        metrics,
        pre_failure_lookback_runs,
    );
    let lead_time_explanation = build_lead_time_explanation(&lead_time_comparison);
    let episode_precision_metrics = build_episode_precision_metrics(metrics, optimized_dsa);
    let executive_summary_text = format!(
        "Primary operator result: DSA episode precision is {:.1}% ({}/{} episodes preceding labeled failures) versus a raw-boundary precision proxy of {:.2}%, a {:.1}x gain. Investigation-worthy burden falls by {:.1}% versus numeric-only DSA ({} -> {} points), raw boundary episodes collapse by {:.1}% ({} -> {}), and recall reaches {}/{} while pass-run nuisance reduction versus EWMA remains bounded at {:.1}%.",
        episode_precision_metrics.dsfb_precision * 100.0,
        episode_precision_metrics.dsfb_pre_failure_episode_count,
        episode_precision_metrics.dsfb_episode_count,
        episode_precision_metrics.raw_alarm_precision * 100.0,
        episode_precision_metrics.precision_gain_factor,
        optimization.operator_delta_targets.delta_investigation_load * 100.0,
        optimization.operator_delta_targets.baseline_investigation_points,
        optimization.operator_delta_targets.optimized_review_escalate_points,
        optimization.operator_delta_targets.delta_episode_count * 100.0,
        optimization.operator_delta_targets.baseline_episode_count,
        optimization.operator_delta_targets.optimized_episode_count,
        optimization.operator_delta_targets.selected_configuration.failure_recall,
        optimization.operator_delta_targets.selected_configuration.failure_runs,
        optimization.operator_delta_targets.delta_nuisance_vs_ewma * 100.0,
    );
    let paper_abstract_artifact = format!(
        "On SECOM, the policy-governed DSFB layer compresses raw structural activity from {} boundary episodes to {} DSA episodes, reduces investigation-worthy feature points from {} numeric-only DSA points to {}, and raises episode precision to {:.1}% from a raw-boundary precision proxy of {:.2}%. This supports a bounded structural-compression claim, not a blanket early-warning or nuisance-superiority claim.",
        optimization.operator_delta_targets.baseline_episode_count,
        optimization.operator_delta_targets.optimized_episode_count,
        optimization.operator_delta_targets.baseline_investigation_points,
        optimization.operator_delta_targets.optimized_review_escalate_points,
        episode_precision_metrics.dsfb_precision * 100.0,
        episode_precision_metrics.raw_alarm_precision * 100.0,
    );

    SecomAddendumArtifacts {
        recurrent_boundary_stats,
        recurrent_boundary_tradeoff_curve,
        required_tradeoff_statement,
        required_tradeoff_statement_supported,
        metric_regrounding,
        target_d_regression_analysis,
        missed_failure_root_cause,
        lead_time_comparison,
        lead_time_explanation,
        episode_precision_metrics,
        executive_summary_text,
        paper_abstract_artifact,
    }
}

pub fn draw_recurrent_boundary_tradeoff_plot(
    path: &Path,
    rows: &[RecurrentBoundaryTradeoffRow],
) -> Result<()> {
    let root =
        BitMapBackend::new(path, (TRADEOFF_PLOT_WIDTH, TRADEOFF_PLOT_HEIGHT)).into_drawing_area();
    root.fill(&WHITE).map_err(plot_error)?;

    let mut chart = ChartBuilder::on(&root)
        .caption(
            "SECOM recurrent_boundary_approach suppression tradeoff",
            ("sans-serif", 28),
        )
        .margin(20)
        .x_label_area_size(60)
        .y_label_area_size(70)
        .build_cartesian_2d(0usize..rows.len(), 0.0f64..1.05f64)
        .map_err(plot_error)?;

    chart
        .configure_mesh()
        .disable_mesh()
        .x_labels(rows.len())
        .x_label_formatter(&|index| {
            rows.get(*index)
                .map(|row| row.suppression_label.clone())
                .unwrap_or_default()
        })
        .y_desc("Recall rate / nuisance reduction")
        .draw()
        .map_err(plot_error)?;

    let recall_series = rows
        .iter()
        .enumerate()
        .map(|(index, row)| (index, row.recall_rate))
        .collect::<Vec<_>>();
    let nuisance_series = rows
        .iter()
        .enumerate()
        .map(|(index, row)| (index, row.delta_nuisance_vs_ewma))
        .collect::<Vec<_>>();

    chart
        .draw_series(LineSeries::new(recall_series.clone(), &BLACK))
        .map_err(plot_error)?;
    chart
        .draw_series(
            recall_series
                .into_iter()
                .map(|point| Circle::new(point, 5, BLACK.filled())),
        )
        .map_err(plot_error)?;

    chart
        .draw_series(LineSeries::new(
            nuisance_series.clone(),
            &RGBColor(100, 100, 100),
        ))
        .map_err(plot_error)?;
    chart
        .draw_series(
            nuisance_series
                .into_iter()
                .map(|point| TriangleMarker::new(point, 6, RGBColor(100, 100, 100).filled())),
        )
        .map_err(plot_error)?;

    chart
        .draw_series(std::iter::once(PathElement::new(
            vec![(0usize, 0.40f64), (rows.len().saturating_sub(1), 0.40f64)],
            ShapeStyle::from(&RGBColor(170, 170, 170)).stroke_width(1),
        )))
        .map_err(plot_error)?;

    root.present().map_err(plot_error)?;
    Ok(())
}

fn build_recurrent_boundary_stats(
    dataset: &PreparedDataset,
    motifs: &MotifSet,
    pre_failure_lookback_runs: usize,
) -> RecurrentBoundaryStats {
    let failure_mask = failure_window_mask(
        dataset.labels.len(),
        &dataset.labels,
        pre_failure_lookback_runs,
    );
    let total_boundary_points = motifs
        .traces
        .iter()
        .flat_map(|trace| trace.labels.iter())
        .filter(|label| **label == DsfbMotifClass::RecurrentBoundaryApproach)
        .count();
    let mut run_hits = BTreeSet::new();
    let mut pre_failure_hits = BTreeSet::new();
    let mut pass_run_hits = BTreeSet::new();

    for trace in &motifs.traces {
        for (run_index, label) in trace.labels.iter().enumerate() {
            if *label != DsfbMotifClass::RecurrentBoundaryApproach {
                continue;
            }
            run_hits.insert(run_index);
            if failure_mask[run_index] {
                pre_failure_hits.insert(run_index);
            }
            if dataset.labels[run_index] == -1 {
                pass_run_hits.insert(run_index);
            }
        }
    }

    let total_run_hits = run_hits.len();
    let total_pre_failure_hits = pre_failure_hits.len();
    let pass_run_hits_count = pass_run_hits.len();

    RecurrentBoundaryStats {
        total_boundary_points,
        total_run_hits,
        total_pre_failure_hits,
        pass_run_hits: pass_run_hits_count,
        precision_pre_failure: ratio(total_pre_failure_hits, total_run_hits),
        precision_pass: ratio(pass_run_hits_count, total_run_hits),
    }
}

fn build_recurrent_boundary_tradeoff_curve(
    dataset: &PreparedDataset,
    grammar: &GrammarSet,
    motifs: &MotifSet,
    metrics: &BenchmarkMetrics,
    optimized_dsa: &DsaEvaluation,
    pre_failure_lookback_runs: usize,
) -> Vec<RecurrentBoundaryTradeoffRow> {
    let motif_by_feature = motifs
        .traces
        .iter()
        .map(|trace| (trace.feature_index, trace))
        .collect::<BTreeMap<_, _>>();
    let grammar_by_feature = grammar
        .traces
        .iter()
        .map(|trace| (trace.feature_index, trace))
        .collect::<BTreeMap<_, _>>();
    let pass_indices = dataset
        .labels
        .iter()
        .enumerate()
        .filter_map(|(index, label)| (*label == -1).then_some(index))
        .collect::<Vec<_>>();
    let failure_indices = dataset
        .labels
        .iter()
        .enumerate()
        .filter_map(|(index, label)| (*label == 1).then_some(index))
        .collect::<Vec<_>>();

    let suppression_levels = [
        (0.0, "baseline"),
        (0.25, "mild_gate"),
        (0.50, "watch_cap"),
        (0.75, "strong_cap"),
        (1.00, "silent_cap"),
    ];

    suppression_levels
        .into_iter()
        .map(|(level, label)| {
            let mut feature_alerts =
                vec![vec![false; dataset.labels.len()]; optimized_dsa.traces.len()];
            let mut investigation_points = 0usize;

            for (trace_slot, feature_trace) in optimized_dsa.traces.iter().enumerate() {
                let motif_trace = motif_by_feature
                    .get(&feature_trace.feature_index)
                    .unwrap_or_else(|| {
                        panic!("missing motif trace {}", feature_trace.feature_index)
                    });
                let grammar_trace = grammar_by_feature
                    .get(&feature_trace.feature_index)
                    .unwrap_or_else(|| {
                        panic!("missing grammar trace {}", feature_trace.feature_index)
                    });
                for run_index in 0..feature_trace.policy_state.len() {
                    let remapped = remap_recurrent_boundary_state(
                        level,
                        feature_trace.policy_state[run_index],
                        motif_trace.labels[run_index],
                        grammar_trace.raw_states[run_index],
                        grammar_trace.persistent_violation[run_index],
                        feature_trace.boundary_density_w[run_index],
                        feature_trace.motif_recurrence_w[run_index],
                    );
                    if is_review_or_escalate(remapped) {
                        feature_alerts[trace_slot][run_index] = true;
                        investigation_points += 1;
                    }
                }
            }

            let corroborating_m = optimized_dsa.run_signals.corroborating_feature_count_min;
            let run_alert = (0..dataset.labels.len())
                .map(|run_index| {
                    feature_alerts
                        .iter()
                        .filter(|trace| trace[run_index])
                        .count()
                        >= corroborating_m
                })
                .collect::<Vec<_>>();

            let failure_recall = failure_indices
                .iter()
                .filter(|&&failure_index| {
                    let start = failure_index.saturating_sub(pre_failure_lookback_runs);
                    run_alert[start..failure_index].iter().any(|flag| *flag)
                })
                .count();
            let pass_alert_runs = pass_indices
                .iter()
                .filter(|&&run_index| run_alert[run_index])
                .count();

            RecurrentBoundaryTradeoffRow {
                suppression_level: level,
                suppression_label: label.into(),
                investigation_points,
                pass_run_nuisance_proxy: ratio(pass_alert_runs, pass_indices.len()),
                delta_nuisance_vs_selected_dsa: relative_reduction(
                    optimized_dsa.summary.pass_run_nuisance_proxy,
                    ratio(pass_alert_runs, pass_indices.len()),
                ),
                delta_nuisance_vs_ewma: relative_reduction(
                    metrics.summary.pass_run_ewma_nuisance_rate,
                    ratio(pass_alert_runs, pass_indices.len()),
                ),
                failure_recall,
                failure_runs: failure_indices.len(),
                recall_rate: ratio(failure_recall, failure_indices.len()),
            }
        })
        .collect()
}

fn recurrent_boundary_tradeoff_statement_supported(rows: &[RecurrentBoundaryTradeoffRow]) -> bool {
    rows.iter().any(|row| row.delta_nuisance_vs_ewma >= 0.40)
        && rows
            .iter()
            .filter(|row| row.delta_nuisance_vs_ewma >= 0.40)
            .all(|row| row.failure_recall < row.failure_runs)
}

fn remap_recurrent_boundary_state(
    suppression_level: f64,
    state: DsaPolicyState,
    motif: DsfbMotifClass,
    grammar_state: GrammarState,
    persistent_violation: bool,
    boundary_density: f64,
    motif_recurrence: f64,
) -> DsaPolicyState {
    if motif != DsfbMotifClass::RecurrentBoundaryApproach {
        return state;
    }

    if suppression_level >= 1.0 {
        return DsaPolicyState::Silent;
    }
    if suppression_level >= 0.75 {
        if persistent_violation || grammar_state == GrammarState::Violation {
            return DsaPolicyState::Watch;
        }
        return DsaPolicyState::Silent;
    }
    if suppression_level >= 0.50 {
        if is_review_or_escalate(state) {
            return DsaPolicyState::Watch;
        }
        return state;
    }
    if suppression_level >= 0.25
        && is_review_or_escalate(state)
        && (boundary_density < 0.75 || motif_recurrence < 0.75)
    {
        return DsaPolicyState::Watch;
    }

    state
}

fn build_metric_regrounding(
    dataset: &PreparedDataset,
    residuals: &ResidualSet,
    baselines: &BaselineSet,
    metrics: &BenchmarkMetrics,
    optimized_dsa: &DsaEvaluation,
    pre_failure_lookback_runs: usize,
) -> Vec<MetricRegroundingRow> {
    let threshold_signal = (0..dataset.labels.len())
        .map(|run_index| {
            residuals
                .traces
                .iter()
                .any(|trace| trace.threshold_alarm[run_index])
        })
        .collect::<Vec<_>>();
    let ewma_signal = (0..dataset.labels.len())
        .map(|run_index| baselines.ewma.iter().any(|trace| trace.alarm[run_index]))
        .collect::<Vec<_>>();

    let threshold_episode_count = count_episodes(&threshold_signal);
    let threshold_precision = episode_precision(
        &threshold_signal,
        &failure_window_mask(
            dataset.labels.len(),
            &dataset.labels,
            pre_failure_lookback_runs,
        ),
    );

    let ewma_episode_count = count_episodes(&ewma_signal);
    let ewma_precision = episode_precision(
        &ewma_signal,
        &failure_window_mask(
            dataset.labels.len(),
            &dataset.labels,
            pre_failure_lookback_runs,
        ),
    );

    let failure_mask = failure_window_mask(
        dataset.labels.len(),
        &dataset.labels,
        pre_failure_lookback_runs,
    );
    let numeric_dsa_episode_count =
        count_episodes(&optimized_dsa.run_signals.numeric_primary_run_alert);
    let dsfb_precision = optimized_dsa
        .episode_summary
        .precursor_quality
        .unwrap_or_default();
    let numeric_precision = episode_precision(
        &optimized_dsa.run_signals.numeric_primary_run_alert,
        &failure_mask,
    );

    vec![
        MetricRegroundingRow {
            metric: "Investigation load".into(),
            baseline: "Numeric-only DSA".into(),
            dsfb_value: investigation_points(optimized_dsa) as f64,
            baseline_value: optimized_dsa.summary.numeric_alert_point_count as f64,
            delta_percent: relative_reduction(
                optimized_dsa.summary.numeric_alert_point_count as f64,
                investigation_points(optimized_dsa) as f64,
            ),
        },
        MetricRegroundingRow {
            metric: "Investigation load".into(),
            baseline: "Threshold".into(),
            dsfb_value: investigation_points(optimized_dsa) as f64,
            baseline_value: metrics.summary.threshold_alarm_points as f64,
            delta_percent: relative_reduction(
                metrics.summary.threshold_alarm_points as f64,
                investigation_points(optimized_dsa) as f64,
            ),
        },
        MetricRegroundingRow {
            metric: "Investigation load".into(),
            baseline: "EWMA".into(),
            dsfb_value: investigation_points(optimized_dsa) as f64,
            baseline_value: metrics.summary.ewma_alarm_points as f64,
            delta_percent: relative_reduction(
                metrics.summary.ewma_alarm_points as f64,
                investigation_points(optimized_dsa) as f64,
            ),
        },
        MetricRegroundingRow {
            metric: "Episode count".into(),
            baseline: "Numeric-only DSA".into(),
            dsfb_value: optimized_dsa.episode_summary.dsa_episode_count as f64,
            baseline_value: numeric_dsa_episode_count as f64,
            delta_percent: relative_reduction(
                numeric_dsa_episode_count as f64,
                optimized_dsa.episode_summary.dsa_episode_count as f64,
            ),
        },
        MetricRegroundingRow {
            metric: "Episode count".into(),
            baseline: "Threshold".into(),
            dsfb_value: optimized_dsa.episode_summary.dsa_episode_count as f64,
            baseline_value: threshold_episode_count as f64,
            delta_percent: relative_reduction(
                threshold_episode_count as f64,
                optimized_dsa.episode_summary.dsa_episode_count as f64,
            ),
        },
        MetricRegroundingRow {
            metric: "Episode count".into(),
            baseline: "EWMA".into(),
            dsfb_value: optimized_dsa.episode_summary.dsa_episode_count as f64,
            baseline_value: ewma_episode_count as f64,
            delta_percent: relative_reduction(
                ewma_episode_count as f64,
                optimized_dsa.episode_summary.dsa_episode_count as f64,
            ),
        },
        MetricRegroundingRow {
            metric: "Nuisance reduction".into(),
            baseline: "Numeric-only DSA".into(),
            dsfb_value: optimized_dsa.summary.pass_run_nuisance_proxy,
            baseline_value: optimized_dsa
                .summary
                .numeric_primary_pass_run_nuisance_proxy,
            delta_percent: relative_reduction(
                optimized_dsa
                    .summary
                    .numeric_primary_pass_run_nuisance_proxy,
                optimized_dsa.summary.pass_run_nuisance_proxy,
            ),
        },
        MetricRegroundingRow {
            metric: "Nuisance reduction".into(),
            baseline: "Threshold".into(),
            dsfb_value: optimized_dsa.summary.pass_run_nuisance_proxy,
            baseline_value: metrics.summary.pass_run_threshold_nuisance_rate,
            delta_percent: relative_reduction(
                metrics.summary.pass_run_threshold_nuisance_rate,
                optimized_dsa.summary.pass_run_nuisance_proxy,
            ),
        },
        MetricRegroundingRow {
            metric: "Nuisance reduction".into(),
            baseline: "EWMA".into(),
            dsfb_value: optimized_dsa.summary.pass_run_nuisance_proxy,
            baseline_value: metrics.summary.pass_run_ewma_nuisance_rate,
            delta_percent: relative_reduction(
                metrics.summary.pass_run_ewma_nuisance_rate,
                optimized_dsa.summary.pass_run_nuisance_proxy,
            ),
        },
        MetricRegroundingRow {
            metric: "Recall".into(),
            baseline: "Numeric-only DSA".into(),
            dsfb_value: optimized_dsa.summary.failure_run_recall as f64,
            baseline_value: optimized_dsa.summary.numeric_primary_failure_run_recall as f64,
            delta_percent: relative_gain(
                optimized_dsa.summary.numeric_primary_failure_run_recall as f64,
                optimized_dsa.summary.failure_run_recall as f64,
            ),
        },
        MetricRegroundingRow {
            metric: "Recall".into(),
            baseline: "Threshold".into(),
            dsfb_value: optimized_dsa.summary.failure_run_recall as f64,
            baseline_value: metrics.summary.failure_runs_with_preceding_threshold_signal as f64,
            delta_percent: relative_gain(
                metrics.summary.failure_runs_with_preceding_threshold_signal as f64,
                optimized_dsa.summary.failure_run_recall as f64,
            ),
        },
        MetricRegroundingRow {
            metric: "Recall".into(),
            baseline: "EWMA".into(),
            dsfb_value: optimized_dsa.summary.failure_run_recall as f64,
            baseline_value: metrics.summary.failure_runs_with_preceding_ewma_signal as f64,
            delta_percent: relative_gain(
                metrics.summary.failure_runs_with_preceding_ewma_signal as f64,
                optimized_dsa.summary.failure_run_recall as f64,
            ),
        },
        MetricRegroundingRow {
            metric: "Lead time".into(),
            baseline: "Numeric-only DSA".into(),
            dsfb_value: optimized_dsa
                .summary
                .mean_lead_time_runs
                .unwrap_or_default(),
            baseline_value: optimized_dsa
                .comparison_summary
                .numeric_dsa
                .mean_lead_time_runs
                .unwrap_or_default(),
            delta_percent: relative_gain(
                optimized_dsa
                    .comparison_summary
                    .numeric_dsa
                    .mean_lead_time_runs
                    .unwrap_or_default(),
                optimized_dsa
                    .summary
                    .mean_lead_time_runs
                    .unwrap_or_default(),
            ),
        },
        MetricRegroundingRow {
            metric: "Lead time".into(),
            baseline: "Threshold".into(),
            dsfb_value: optimized_dsa
                .summary
                .mean_lead_time_runs
                .unwrap_or_default(),
            baseline_value: metrics
                .lead_time_summary
                .mean_threshold_lead_runs
                .unwrap_or_default(),
            delta_percent: relative_gain(
                metrics
                    .lead_time_summary
                    .mean_threshold_lead_runs
                    .unwrap_or_default(),
                optimized_dsa
                    .summary
                    .mean_lead_time_runs
                    .unwrap_or_default(),
            ),
        },
        MetricRegroundingRow {
            metric: "Lead time".into(),
            baseline: "EWMA".into(),
            dsfb_value: optimized_dsa
                .summary
                .mean_lead_time_runs
                .unwrap_or_default(),
            baseline_value: metrics
                .lead_time_summary
                .mean_ewma_lead_runs
                .unwrap_or_default(),
            delta_percent: relative_gain(
                metrics
                    .lead_time_summary
                    .mean_ewma_lead_runs
                    .unwrap_or_default(),
                optimized_dsa
                    .summary
                    .mean_lead_time_runs
                    .unwrap_or_default(),
            ),
        },
        MetricRegroundingRow {
            metric: "Episode precision".into(),
            baseline: "Numeric-only DSA".into(),
            dsfb_value: dsfb_precision,
            baseline_value: numeric_precision,
            delta_percent: relative_gain(numeric_precision, dsfb_precision),
        },
        MetricRegroundingRow {
            metric: "Episode precision".into(),
            baseline: "Threshold".into(),
            dsfb_value: dsfb_precision,
            baseline_value: threshold_precision,
            delta_percent: relative_gain(threshold_precision, dsfb_precision),
        },
        MetricRegroundingRow {
            metric: "Episode precision".into(),
            baseline: "EWMA".into(),
            dsfb_value: dsfb_precision,
            baseline_value: ewma_precision,
            delta_percent: relative_gain(ewma_precision, dsfb_precision),
        },
    ]
}

fn build_target_d_regression_analysis(
    dataset: &PreparedDataset,
    grammar: &GrammarSet,
    motifs: &MotifSet,
    failure_driven: &FailureDrivenArtifacts,
    baseline_dsa: &DsaEvaluation,
    optimized_dsa: &DsaEvaluation,
    tradeoff_curve: &[RecurrentBoundaryTradeoffRow],
) -> TargetDRegressionAnalysis {
    let pass_indices = dataset
        .labels
        .iter()
        .enumerate()
        .filter_map(|(index, label)| (*label == -1).then_some(index))
        .collect::<Vec<_>>();
    let motif_by_feature = motifs
        .traces
        .iter()
        .map(|trace| (trace.feature_index, trace))
        .collect::<BTreeMap<_, _>>();
    let grammar_by_feature = grammar
        .traces
        .iter()
        .map(|trace| (trace.feature_index, trace))
        .collect::<BTreeMap<_, _>>();

    let mut added_pass_points_by_feature = BTreeMap::<String, usize>::new();
    let mut motif_counts = BTreeMap::<String, usize>::new();
    let mut grammar_counts = BTreeMap::<String, usize>::new();

    for (baseline_trace, optimized_trace) in baseline_dsa.traces.iter().zip(&optimized_dsa.traces) {
        let motif_trace = match motif_by_feature.get(&optimized_trace.feature_index) {
            Some(trace) => *trace,
            None => continue,
        };
        let grammar_trace = match grammar_by_feature.get(&optimized_trace.feature_index) {
            Some(trace) => *trace,
            None => continue,
        };
        let mut added = 0usize;
        for &run_index in &pass_indices {
            let baseline_flag = is_review_or_escalate(baseline_trace.policy_state[run_index]);
            let optimized_flag = is_review_or_escalate(optimized_trace.policy_state[run_index]);
            if optimized_flag && !baseline_flag {
                added += 1;
                *motif_counts
                    .entry(motif_trace.labels[run_index].as_lowercase().to_string())
                    .or_default() += 1;
                *grammar_counts
                    .entry(grammar_label(grammar_trace.raw_states[run_index]).to_string())
                    .or_default() += 1;
            }
        }
        if added > 0 {
            added_pass_points_by_feature.insert(optimized_trace.feature_name.clone(), added);
        }
    }

    let mut contributing_features_counts =
        added_pass_points_by_feature.into_iter().collect::<Vec<_>>();
    contributing_features_counts
        .sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(&right.0)));
    let contributing_features = contributing_features_counts
        .into_iter()
        .take(5)
        .map(|(feature, _)| feature)
        .collect::<Vec<_>>();

    let mut contributing_motifs_counts = motif_counts.into_iter().collect::<Vec<_>>();
    contributing_motifs_counts
        .sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(&right.0)));
    let contributing_motifs = contributing_motifs_counts
        .into_iter()
        .take(3)
        .map(|(motif, _)| motif)
        .collect::<Vec<_>>();

    let contributing_features_text = if contributing_features.is_empty() {
        "no single dominant feature".into()
    } else {
        contributing_features.join(", ")
    };
    let contributing_motifs_text = if contributing_motifs.is_empty() {
        "none".into()
    } else {
        contributing_motifs.join(", ")
    };
    let grammar_counts_text = if grammar_counts.is_empty() {
        "none".into()
    } else {
        grammar_counts
            .clone()
            .into_iter()
            .take(3)
            .map(|(name, _)| name)
            .collect::<Vec<_>>()
            .join(", ")
    };

    let mut contributing_heuristics = failure_driven
        .heuristic_provenance
        .iter()
        .filter(|row| {
            row.intended_effect == "recover_failure"
                && contributing_features
                    .iter()
                    .any(|feature| row.uses_features.contains(feature))
        })
        .map(|row| row.heuristic_id.clone())
        .collect::<Vec<_>>();
    contributing_heuristics.sort();
    contributing_heuristics.dedup();

    let mut contributing_policy_rules = Vec::new();
    if contributing_features
        .iter()
        .any(|feature| matches!(feature.as_str(), "S092" | "S134" | "S275"))
    {
        contributing_policy_rules.push(
            "bounded recall-rescue override: allow_review_without_escalate=true with persistence and corroboration relaxed on feature-local recovery paths"
                .into(),
        );
    }
    if optimized_dsa
        .parameter_manifest
        .feature_policy_override_summary
        .iter()
        .any(|summary| summary.contains("suppress_if_isolated=true"))
    {
        contributing_policy_rules.push(
            "isolated-pass suppression rule: allow_watch_only=true plus suppress_if_isolated=true on nuisance-only pass features"
                .into(),
        );
    }
    if contributing_motifs
        .iter()
        .any(|motif| motif == "recurrent_boundary_approach")
    {
        contributing_policy_rules.push(
            "recurrent_boundary_approach remains persistence-gated and globally capped below strong suppression because stronger nuisance cuts give back recall"
                .into(),
        );
    }

    let strong_suppression_rows = tradeoff_curve
        .iter()
        .filter(|row| row.delta_nuisance_vs_ewma >= 0.40)
        .collect::<Vec<_>>();
    let tradeoff_justification = if strong_suppression_rows.is_empty() {
        "No recurrent_boundary_approach suppression row reached a 40% nuisance reduction versus EWMA, so the current regression is carried as a bounded operator tradeoff rather than falsely claimed as solved.".into()
    } else {
        let recall_rows = strong_suppression_rows
            .iter()
            .map(|row| {
                format!(
                    "{} => recall {}/{}",
                    row.suppression_label, row.failure_recall, row.failure_runs
                )
            })
            .collect::<Vec<_>>()
            .join("; ");
        format!(
            "Further suppression was rejected in this pass. Every recurrent_boundary_approach suppression row that reaches at least a 40% nuisance reduction versus EWMA drops recall below full coverage: {}.",
            recall_rows
        )
    };

    TargetDRegressionAnalysis {
        contributing_features,
        contributing_motifs,
        contributing_heuristics,
        contributing_policy_rules,
        causal_chain: vec![
            "Failure-local rescue overrides for the former misses promote grammar-qualified Watch states to Review on specific low-burden features.".into(),
            "Those promotions create a small number of additional pass-only run segments, which increases review episodes per pass run even while total investigation points still fall materially.".into(),
            "The accepted isolated-pass suppressions reduce point burden, but they do not fully remove the extra run segmentation introduced by recall recovery.".into(),
            "The recurrent_boundary_approach suppression sweep shows that stronger motif-wide suppression would cut nuisance further only by giving back recall.".into(),
        ],
        why_regression_occurred: format!(
            "The regression is driven by recall-recovery promotions on {} under motif classes {} and grammar states {}. The specific policy change is bounded Watch->Review rescue without escalation on these features, while the compensating isolated-pass suppression rules only partially offset the extra pass-only segmentation.",
            contributing_features_text,
            contributing_motifs_text,
            grammar_counts_text,
        ),
        action_taken: "formal_tradeoff_justification".into(),
        tradeoff_justification,
    }
}

fn build_missed_failure_root_cause(
    failure_driven: &FailureDrivenArtifacts,
) -> MissedFailureRootCause {
    let case = failure_driven
        .failure_cases
        .iter()
        .find(|case| case.failure_id == 2)
        .or_else(|| {
            failure_driven
                .failure_cases
                .iter()
                .find(|case| !case.baseline_detected_by_dsa && case.optimized_detected_by_dsa)
        })
        .or_else(|| failure_driven.failure_cases.first())
        .unwrap_or_else(|| panic!("failure-driven artifacts missing baseline failure cases"));
    let top_feature = case
        .top_contributing_features
        .first()
        .unwrap_or_else(|| panic!("failure case {} missing top feature", case.failure_id));
    let reason_for_miss = missed_failure_reason(case.exact_miss_rule.as_str()).to_string();
    let classification = if case.optimized_detected_by_dsa {
        "recoverable (and fix)"
    } else if top_feature.max_dsa_score <= 0.0 {
        "data limitation"
    } else {
        "structurally unrecoverable"
    };

    MissedFailureRootCause {
        failure_id: case.failure_id,
        feature_activity: case
            .top_contributing_features
            .iter()
            .map(|feature| MissedFailureFeatureActivity {
                feature_name: feature.feature_name.clone(),
                behavior_classification: feature.behavior_classification.clone(),
                max_dsa_score: feature.max_dsa_score,
                initial_motif_hypothesis: feature.initial_motif_hypothesis.clone(),
                dominant_dsfb_motif: feature.dominant_dsfb_motif.clone(),
                dominant_grammar_state: feature.dominant_grammar_state.clone(),
                failure_explanation: feature.failure_explanation.clone(),
            })
            .collect(),
        residual_trajectory: top_feature.residual_trajectory.clone(),
        drift_trajectory: top_feature.drift_trajectory.clone(),
        slew_trajectory: top_feature.slew_trajectory.clone(),
        motif_presence: top_feature.motif_timeline.clone(),
        grammar_state: top_feature.grammar_state_timeline.clone(),
        reason_for_miss,
        classification: classification.into(),
        recovered_after_fix: case.optimized_detected_by_dsa,
        recovery_feature: case
            .top_contributing_features
            .iter()
            .find(|feature| feature.feature_name == "S092")
            .map(|feature| feature.feature_name.clone())
            .or_else(|| {
                case.top_contributing_features
                    .first()
                    .map(|feature| feature.feature_name.clone())
            }),
        recovery_note: format!(
            "The former 103/104 limiting case was failure run {}. It remained a Watch-class near-miss under {} until the bounded {} structural-support rescue was accepted, after which the selected row reached 104/104.",
            case.failure_id,
            case.exact_miss_rule,
            case
                .top_contributing_features
                .iter()
                .find(|feature| feature.feature_name == "S092")
                .map(|feature| feature.feature_name.as_str())
                .unwrap_or(top_feature.feature_name.as_str()),
        ),
    }
}

fn build_lead_time_comparison(
    semantic_layer: &SemanticLayer,
    optimized_dsa: &DsaEvaluation,
    metrics: &BenchmarkMetrics,
    pre_failure_lookback_runs: usize,
) -> Vec<LeadTimeComparisonRow> {
    let dsa_by_failure = optimized_dsa
        .per_failure_run_signals
        .iter()
        .map(|row| (row.failure_run_index, row))
        .collect::<BTreeMap<_, _>>();

    metrics
        .per_failure_run_signals
        .iter()
        .map(|metric_row| {
            let dsa_row = dsa_by_failure
                .get(&metric_row.failure_run_index)
                .copied()
                .unwrap_or_else(|| {
                    panic!("missing DSA failure row {}", metric_row.failure_run_index)
                });
            let earliest_semantic_match_run = earliest_semantic_match_run(
                semantic_layer,
                metric_row.failure_run_index,
                pre_failure_lookback_runs,
            );
            let earliest_semantic_match_lead_runs = earliest_semantic_match_run
                .map(|run_index| metric_row.failure_run_index - run_index);
            LeadTimeComparisonRow {
                failure_id: metric_row.failure_run_index,
                dsfb_lead_runs: dsa_row.dsa_lead_runs,
                threshold_lead_runs: metric_row.threshold_lead_runs,
                earliest_semantic_match_lead_runs,
                threshold_minus_dsfb_runs: diff_optional(
                    metric_row.threshold_lead_runs,
                    dsa_row.dsa_lead_runs,
                ),
                threshold_minus_semantic_match_runs: diff_optional(
                    metric_row.threshold_lead_runs,
                    earliest_semantic_match_lead_runs,
                ),
            }
        })
        .collect()
}

fn build_lead_time_explanation(rows: &[LeadTimeComparisonRow]) -> LeadTimeExplanation {
    let mean_dsfb = mean_option(
        rows.iter()
            .filter_map(|row| row.dsfb_lead_runs.map(|value| value as f64)),
    );
    let mean_threshold = mean_option(
        rows.iter()
            .filter_map(|row| row.threshold_lead_runs.map(|value| value as f64)),
    );
    let mean_semantic = mean_option(rows.iter().filter_map(|row| {
        row.earliest_semantic_match_lead_runs
            .map(|value| value as f64)
    }));
    let threshold_earlier_failure_count = rows
        .iter()
        .filter(|row| {
            matches!(
                (row.threshold_lead_runs, row.dsfb_lead_runs),
                (Some(threshold), Some(dsfb)) if threshold > dsfb
            )
        })
        .count();
    let dsfb_earlier_failure_count = rows
        .iter()
        .filter(|row| {
            matches!(
                (row.threshold_lead_runs, row.dsfb_lead_runs),
                (Some(threshold), Some(dsfb)) if dsfb > threshold
            )
        })
        .count();
    let semantic_match_precedes_threshold_count = rows
        .iter()
        .filter(|row| {
            matches!(
                (row.earliest_semantic_match_lead_runs, row.threshold_lead_runs),
                (Some(semantic), Some(threshold)) if semantic > threshold
            )
        })
        .count();

    LeadTimeExplanation {
        mean_dsfb_lead_runs: mean_dsfb,
        mean_threshold_lead_runs: mean_threshold,
        mean_semantic_match_lead_runs: mean_semantic,
        threshold_earlier_failure_count,
        dsfb_earlier_failure_count,
        semantic_match_precedes_threshold_count,
        motif_emergence_precedes_threshold_count: semantic_match_precedes_threshold_count,
        explanation: "Threshold fires on any sufficiently large residual deviation, while DSFB waits for grammar-qualified motif structure and then applies persistence-constrained policy promotion. That layered requirement makes DSFB more selective but also later on SECOM.".into(),
        validation_note: format!(
            "Mean threshold lead is {} runs versus mean DSA lead {} runs; earliest grammar-qualified motif emergence averages {} runs. This gap is consistent with DSFB waiting for structured motifs rather than raw deviation alone.",
            format_option(mean_threshold),
            format_option(mean_dsfb),
            format_option(mean_semantic),
        ),
    }
}

fn missed_failure_reason(exact_miss_rule: &str) -> &'static str {
    match exact_miss_rule {
        "watch_class_near_miss_below_numeric_gate" | "policy_state_never_reached_review" => {
            "policy suppression"
        }
        "directional_consistency_gate" | "feature_override_fragmentation_ceiling" => {
            "grammar rejection"
        }
        "numeric_score_below_tau" => "no precursor",
        _ => "feature absence",
    }
}

fn build_episode_precision_metrics(
    metrics: &BenchmarkMetrics,
    optimized_dsa: &DsaEvaluation,
) -> EpisodePrecisionMetrics {
    let dsfb_episode_count = optimized_dsa.episode_summary.dsa_episode_count;
    let dsfb_pre_failure_episode_count =
        optimized_dsa.episode_summary.dsa_episodes_preceding_failure;
    let dsfb_precision = optimized_dsa
        .episode_summary
        .precursor_quality
        .unwrap_or_default();
    let raw_alarm_count = optimized_dsa.episode_summary.raw_boundary_episode_count;
    let raw_alarm_precision = ratio(metrics.summary.failure_runs, raw_alarm_count);
    let precision_gain_factor = if raw_alarm_precision > 0.0 {
        dsfb_precision / raw_alarm_precision
    } else {
        0.0
    };

    EpisodePrecisionMetrics {
        dsfb_episode_count,
        dsfb_pre_failure_episode_count,
        dsfb_precision,
        raw_alarm_count,
        raw_alarm_precision,
        precision_gain_factor,
    }
}

fn earliest_semantic_match_run(
    semantic_layer: &SemanticLayer,
    failure_index: usize,
    pre_failure_lookback_runs: usize,
) -> Option<usize> {
    let start = failure_index.saturating_sub(pre_failure_lookback_runs);
    semantic_layer
        .semantic_matches
        .iter()
        .filter(|row| row.run_index >= start && row.run_index < failure_index)
        .map(|row| row.run_index)
        .min()
}

fn investigation_points(dsa: &DsaEvaluation) -> usize {
    dsa.summary.review_point_count + dsa.summary.escalate_point_count
}

fn count_episodes(signal: &[bool]) -> usize {
    let mut count = 0usize;
    let mut in_episode = false;
    for flag in signal {
        if *flag && !in_episode {
            count += 1;
            in_episode = true;
        } else if !*flag {
            in_episode = false;
        }
    }
    count
}

fn episode_precision(signal: &[bool], failure_window_mask: &[bool]) -> f64 {
    let episodes = episode_ranges(signal);
    if episodes.is_empty() {
        return 0.0;
    }
    let preceding = episodes
        .iter()
        .filter(|(start, end)| (*start..=*end).any(|index| failure_window_mask[index]))
        .count();
    preceding as f64 / episodes.len() as f64
}

fn episode_ranges(signal: &[bool]) -> Vec<(usize, usize)> {
    let mut episodes = Vec::new();
    let mut start = None;
    for (index, flag) in signal.iter().copied().enumerate() {
        match (start, flag) {
            (None, true) => start = Some(index),
            (Some(episode_start), false) => {
                episodes.push((episode_start, index - 1));
                start = None;
            }
            _ => {}
        }
    }
    if let Some(episode_start) = start {
        episodes.push((episode_start, signal.len().saturating_sub(1)));
    }
    episodes
}

fn failure_window_mask(run_count: usize, labels: &[i8], lookback: usize) -> Vec<bool> {
    let mut mask = vec![false; run_count];
    for (failure_index, label) in labels.iter().enumerate() {
        if *label != 1 {
            continue;
        }
        let start = failure_index.saturating_sub(lookback);
        for slot in &mut mask[start..failure_index] {
            *slot = true;
        }
    }
    mask
}

fn grammar_label(state: GrammarState) -> &'static str {
    match state {
        GrammarState::Admissible => "Admissible",
        GrammarState::Boundary => "BoundaryGrazing",
        GrammarState::Violation => "PersistentViolation",
    }
}

fn is_review_or_escalate(state: DsaPolicyState) -> bool {
    matches!(state, DsaPolicyState::Review | DsaPolicyState::Escalate)
}

fn relative_reduction(baseline: f64, value: f64) -> f64 {
    if baseline.abs() <= f64::EPSILON {
        0.0
    } else {
        (baseline - value) / baseline
    }
}

fn relative_gain(baseline: f64, value: f64) -> f64 {
    if baseline.abs() <= f64::EPSILON {
        0.0
    } else {
        (value - baseline) / baseline
    }
}

fn ratio(numerator: usize, denominator: usize) -> f64 {
    if denominator == 0 {
        0.0
    } else {
        numerator as f64 / denominator as f64
    }
}

fn diff_optional(left: Option<usize>, right: Option<usize>) -> Option<i64> {
    match (left, right) {
        (Some(lhs), Some(rhs)) => Some(lhs as i64 - rhs as i64),
        _ => None,
    }
}

fn mean_option(values: impl Iterator<Item = f64>) -> Option<f64> {
    let values = values.collect::<Vec<_>>();
    (!values.is_empty()).then_some(values.iter().sum::<f64>() / values.len() as f64)
}

fn format_option(value: Option<f64>) -> String {
    value
        .map(|value| format!("{value:.4}"))
        .unwrap_or_else(|| "n/a".into())
}

fn plot_error<E: std::fmt::Display>(error: E) -> DsfbSemiconductorError {
    DsfbSemiconductorError::DatasetFormat(format!("plot generation failed: {error}"))
}