use crate::baselines::compute_baselines;
use chrono::Utc;
use crate::cohort::{
build_seed_feature_check, compute_rating_delta_forecast, compute_rating_failure_analysis,
run_recall_optimization, write_cohort_results_csv,
write_failure_analysis_md as write_cohort_failure_analysis_md,
write_feature_policy_summary_csv, write_feature_ranking_comparison_csv,
write_feature_ranking_csv, write_heuristic_policy_failure_analysis_md,
write_missed_failure_diagnostics_csv, write_motif_policy_contributions_csv,
write_operator_burden_contributions_csv, write_operator_delta_attainment_matrix_csv,
write_policy_contribution_analysis_csv, write_precursor_quality_csv,
write_recall_critical_features_csv, write_recall_recovery_efficiency_csv,
write_recall_rescue_results_csv, write_single_change_iteration_log_csv,
};
use crate::config::{PipelineConfig, RunConfiguration};
use crate::dataset::phm2018::{support_status as phm_support_status, Phm2018SupportStatus};
use crate::dataset::secom::{self, SecomArchiveLayout};
use crate::error::{DsfbSemiconductorError, Result};
use crate::failure_driven::build_failure_driven_artifacts;
use crate::grammar::evaluate_grammar;
use crate::heuristics::build_heuristics_bank;
use crate::metrics::{
compute_metrics, BenchmarkMetrics, BoundaryEpisodeSummary, DensityMetricRecord, DensitySummary,
LeadTimeSummary, PerFailureRunSignal,
};
use crate::nominal::build_nominal_model;
use crate::non_intrusive::materialize_non_intrusive_artifacts;
use crate::output_paths::{create_timestamped_run_dir, default_output_root};
use crate::plots::{generate_figures, FigureManifest};
use crate::precursor::{
DsaEvaluation, DsaRunSignals, DsaVsBaselinesSummary, PerFailureRunDsaSignal,
};
use crate::preprocessing::prepare_secom;
use crate::report::{write_reports, ReportArtifacts};
use crate::residual::compute_residuals;
use crate::secom_addendum::{
build_secom_addendum_artifacts, draw_recurrent_boundary_tradeoff_plot,
};
use crate::semiotics::{build_scaffold_semiotics, build_semantic_layer, classify_motifs};
use crate::signs::compute_signs;
use crate::traceability::{build_traceability_entries, write_traceability_json, DsfbRunManifest};
use serde::Serialize;
use std::fs::{self, File};
use std::io::Write;
use std::path::{Path, PathBuf};
use zip::write::SimpleFileOptions;
#[derive(Debug, Clone)]
pub struct PaperLockMetrics {
pub episode_count: usize,
pub precision: f64,
pub detected_failures: usize,
pub total_failures: usize,
}
#[derive(Debug, Clone)]
pub struct SecomRunArtifacts {
pub run_dir: PathBuf,
pub report: ReportArtifacts,
pub figures: FigureManifest,
pub metrics_path: PathBuf,
pub manifest_path: PathBuf,
pub zip_path: PathBuf,
pub phm2018_status: Phm2018SupportStatus,
pub paper_lock_metrics: PaperLockMetrics,
}
#[derive(Debug, Clone, Serialize)]
struct ArtifactManifest {
dataset: String,
run_dir: String,
metrics_summary_path: String,
baseline_comparison_summary_path: String,
dsa_vs_baselines_summary_path: String,
dsa_parameter_manifest_path: String,
dsa_grid_results_path: String,
dsa_grid_summary_path: String,
dsa_feature_ranking_path: String,
dsa_feature_ranking_recall_aware_path: String,
dsa_feature_ranking_dsfb_aware_path: String,
dsa_feature_ranking_comparison_path: String,
dsa_seed_feature_check_path: String,
dsa_feature_cohorts_path: String,
dsa_feature_policy_overrides_path: String,
dsa_feature_policy_summary_path: String,
dsa_recall_rescue_results_path: String,
dsa_recall_critical_features_path: String,
dsa_pareto_frontier_path: String,
dsa_stage_a_candidates_path: String,
dsa_stage_b_candidates_path: String,
dsa_missed_failure_diagnostics_path: String,
dsa_delta_target_assessment_path: String,
dsa_cohort_results_path: String,
dsa_cohort_results_recall_aware_path: String,
dsa_cohort_results_dsfb_aware_path: String,
dsa_cohort_summary_path: String,
dsa_cohort_summary_recall_aware_path: String,
dsa_cohort_summary_dsfb_aware_path: String,
dsa_cohort_precursor_quality_path: String,
dsa_cohort_failure_analysis_path: Option<String>,
dsa_heuristic_policy_failure_analysis_path: Option<String>,
dsa_motif_policy_contributions_path: String,
dsa_policy_contribution_analysis_path: String,
dsa_rating_delta_forecast_path: String,
dsa_rating_delta_failure_analysis_path: Option<String>,
lead_time_metrics_path: String,
density_metrics_path: String,
cusum_baseline_path: String,
run_energy_baseline_path: String,
pca_fdc_baseline_path: String,
per_failure_run_signals_path: String,
dsa_metrics_path: String,
dsa_run_signals_path: String,
per_failure_run_dsa_signals_path: String,
dsfb_signs_path: String,
dsfb_feature_signs_path: String,
dsfb_motifs_path: String,
dsfb_motif_labels_per_time_path: String,
dsfb_feature_motif_timeline_path: String,
dsfb_grammar_states_path: String,
dsfb_feature_grammar_states_path: String,
dsfb_envelope_interaction_summary_path: String,
dsfb_heuristics_bank_expanded_path: String,
dsfb_semantic_matches_path: String,
dsfb_semantic_ranked_candidates_path: String,
dsfb_feature_policy_decisions_path: String,
dsfb_traceability_path: String,
dsfb_group_definitions_path: String,
dsfb_group_signs_path: String,
dsfb_group_grammar_states_path: String,
dsfb_group_semantic_matches_path: String,
dsfb_structural_delta_metrics_path: String,
recurrent_boundary_stats_path: String,
recurrent_boundary_tradeoff_curve_path: String,
recurrent_boundary_tradeoff_plot_path: String,
dsfb_metric_regrounding_path: String,
target_d_regression_analysis_path: String,
missed_failure_root_cause_path: String,
lead_time_comparison_path: String,
lead_time_explanation_path: String,
episode_precision_metrics_path: String,
dsfb_episode_summary_path: String,
dsfb_episode_precision_path: String,
dsfb_recall_metrics_path: String,
paper_abstract_artifact_path: String,
dsa_operator_baselines_path: String,
dsa_operator_delta_targets_path: String,
dsa_operator_delta_attainment_matrix_path: String,
dsa_policy_operator_burden_contributions_path: String,
dsa_recall_recovery_efficiency_path: String,
dsfb_single_change_iteration_log_path: String,
optimization_log_path: String,
failures_index_path: String,
missed_failure_priority_path: String,
failure_case_paths: Vec<String>,
feature_motif_grounding_path: String,
feature_to_motif_path: String,
negative_control_report_path: String,
dsfb_heuristics_bank_minimal_path: String,
dsfb_heuristic_provenance_path: String,
policy_decisions_path: String,
policy_burden_summary_path: String,
dsfb_feature_role_validation_path: String,
dsfb_group_validation_path: String,
dsfb_vs_ewma_case_paths: Vec<String>,
dsa_stage1_candidates_path: String,
dsa_stage2_candidates_path: String,
dsa_feature_ranking_burden_aware_path: String,
dsa_cohort_results_burden_aware_path: String,
dsa_cohort_summary_burden_aware_path: String,
secom_archive_layout_path: String,
drsc_trace_path: Option<String>,
drsc_figure_path: Option<String>,
drsc_dsa_combined_trace_path: Option<String>,
drsc_dsa_combined_figure_path: Option<String>,
dsa_focus_trace_path: Option<String>,
dsa_focus_figure_path: Option<String>,
non_intrusive_interface_spec_path: String,
non_intrusive_architecture_png_path: String,
non_intrusive_architecture_svg_path: String,
report_markdown_path: String,
report_tex_path: String,
report_pdf_path: Option<String>,
report_pdf_alias_path: Option<String>,
zip_path: String,
results_zip_path: String,
}
#[derive(Debug, Clone, Serialize)]
struct BaselineComparisonSummary {
dataset: String,
secom_archive_layout_note: String,
feature_count_used_by_crate: usize,
failure_runs: usize,
analyzable_feature_count: usize,
grammar_imputation_suppression_points: usize,
lookback_runs: usize,
failure_run_recall: FailureRunRecallSummary,
pass_run_nuisance_proxy: PassRunNuisanceSummary,
lead_time_summary: LeadTimeSummary,
density_summary: DensitySummary,
boundary_episode_summary: BoundaryEpisodeSummary,
dsa_comparison_summary: Option<DsaVsBaselinesSummary>,
}
#[derive(Debug, Clone, Serialize)]
struct FailureRunRecallSummary {
dsfb_raw_signal: usize,
dsfb_persistent_signal: usize,
dsfb_raw_boundary_signal: usize,
dsfb_persistent_boundary_signal: usize,
dsfb_raw_violation_signal: usize,
dsfb_persistent_violation_signal: usize,
dsfb_dsa_signal: usize,
ewma_signal: usize,
cusum_signal: usize,
run_energy_signal: usize,
pca_fdc_signal: usize,
threshold_signal: usize,
}
#[derive(Debug, Clone, Serialize)]
struct PassRunNuisanceSummary {
dsfb_raw_boundary_signal_runs: usize,
dsfb_persistent_boundary_signal_runs: usize,
dsfb_raw_violation_signal_runs: usize,
dsfb_persistent_violation_signal_runs: usize,
dsfb_dsa_signal_runs: usize,
ewma_signal_runs: usize,
cusum_signal_runs: usize,
run_energy_signal_runs: usize,
pca_fdc_signal_runs: usize,
threshold_signal_runs: usize,
dsfb_raw_boundary_signal_rate: f64,
dsfb_persistent_boundary_signal_rate: f64,
dsfb_raw_violation_signal_rate: f64,
dsfb_persistent_violation_signal_rate: f64,
dsfb_dsa_signal_rate: f64,
ewma_signal_rate: f64,
cusum_signal_rate: f64,
run_energy_signal_rate: f64,
pca_fdc_signal_rate: f64,
threshold_signal_rate: f64,
}
#[derive(Debug, Clone, Serialize)]
struct DsfbEpisodeSummaryRow {
raw_boundary_episodes: usize,
dsfb_episodes: usize,
episode_collapse_fraction: f64,
}
#[derive(Debug, Clone, Serialize)]
struct DsfbEpisodePrecisionRow {
raw_boundary_precision: f64,
dsfb_precision: f64,
precision_gain_factor: f64,
raw_boundary_episodes: usize,
dsfb_episodes: usize,
dsfb_pre_failure_episodes: usize,
}
#[derive(Debug, Clone, Serialize)]
struct DsfbRecallMetricsRow {
detected_failures: usize,
total_failures: usize,
recall: f64,
}
pub fn run_secom_benchmark(
data_root: &Path,
output_root: Option<&Path>,
config: PipelineConfig,
fetch_if_missing: bool,
) -> Result<SecomRunArtifacts> {
config
.validate()
.map_err(DsfbSemiconductorError::DatasetFormat)?;
let paths = if fetch_if_missing {
secom::fetch_if_missing(data_root)?
} else {
secom::ensure_present(data_root)?
};
let secom_archive_layout = secom::inspect_archive_layout(&paths)?;
let dataset = secom::load_from_root(data_root)?;
let prepared = prepare_secom(&dataset, &config)?;
let nominal = build_nominal_model(&prepared, &config);
let residuals = compute_residuals(&prepared, &nominal);
let signs = compute_signs(&prepared, &nominal, &residuals, &config);
let baselines = compute_baselines(&prepared, &nominal, &residuals, &config);
let grammar = evaluate_grammar(&residuals, &signs, &nominal, &config);
let mut metrics = compute_metrics(
&prepared, &nominal, &residuals, &signs, &baselines, &grammar, &config,
);
let heuristics = build_heuristics_bank(&metrics, "SECOM");
let motifs = classify_motifs(
&prepared,
&nominal,
&residuals,
&signs,
&grammar,
config.pre_failure_lookback_runs,
);
let mut semantic_layer = build_semantic_layer(
&prepared,
&residuals,
&signs,
&grammar,
&motifs,
&nominal,
config.pre_failure_lookback_runs,
);
let scaffold_semiotics = build_scaffold_semiotics(
&prepared,
&nominal,
&residuals,
&grammar,
&motifs,
&semantic_layer,
);
let optimization = run_recall_optimization(
&prepared,
&nominal,
&residuals,
&signs,
&baselines,
&grammar,
&metrics,
&semantic_layer,
&scaffold_semiotics,
config.pre_failure_lookback_runs,
)?;
let selected_strategy = optimization
.optimized_execution
.summary
.selected_configuration
.as_ref()
.map(|row| row.ranking_strategy.as_str())
.unwrap_or("compression_biased");
let feature_ranking = match selected_strategy {
"recall_aware" => optimization.recall_aware_feature_ranking.clone(),
"burden_aware" => optimization.burden_aware_feature_ranking.clone(),
"dsfb_aware" => optimization.dsfb_aware_feature_ranking.clone(),
_ => optimization.baseline_feature_ranking.clone(),
};
let feature_cohorts = match selected_strategy {
"recall_aware" => optimization.recall_aware_feature_cohorts.clone(),
"burden_aware" => optimization.burden_aware_feature_cohorts.clone(),
"dsfb_aware" => optimization.dsfb_aware_feature_cohorts.clone(),
_ => optimization.baseline_feature_cohorts.clone(),
};
let seed_feature_check = build_seed_feature_check(&feature_cohorts);
let dsa = optimization.optimized_execution.selected_evaluation.clone();
let dsa_grid_summary = optimization.optimized_execution.grid_summary.clone();
let cohort_summary = optimization.optimized_execution.summary.clone();
metrics.dsa_summary = Some(dsa.summary.clone());
semantic_layer.structural_delta_metrics.episode_precision =
dsa.episode_summary.precursor_quality;
semantic_layer.structural_delta_metrics.compression_ratio =
dsa.episode_summary.compression_ratio;
let rating_delta_forecast =
compute_rating_delta_forecast(&dsa, &metrics, Some(&cohort_summary));
let rating_delta_failure_analysis =
compute_rating_failure_analysis(&dsa, &metrics, Some(&cohort_summary));
let failure_driven = build_failure_driven_artifacts(
&prepared,
&residuals,
&signs,
&baselines,
&grammar,
&motifs,
&semantic_layer,
&scaffold_semiotics,
&metrics,
&optimization.baseline_execution.selected_evaluation,
&dsa,
&optimization.missed_failure_diagnostics,
&optimization.policy_operator_burden_contributions,
config.pre_failure_lookback_runs,
);
let secom_addendum = build_secom_addendum_artifacts(
&prepared,
&residuals,
&baselines,
&grammar,
&motifs,
&semantic_layer,
&metrics,
&optimization,
&failure_driven,
&optimization.baseline_execution.selected_evaluation,
&dsa,
config.pre_failure_lookback_runs,
);
let paper_lock_metrics = PaperLockMetrics {
episode_count: secom_addendum.episode_precision_metrics.dsfb_episode_count,
precision: secom_addendum.episode_precision_metrics.dsfb_precision,
detected_failures: optimization
.operator_delta_targets
.selected_configuration
.failure_recall,
total_failures: optimization
.operator_delta_targets
.selected_configuration
.failure_runs,
};
let output_root = output_root
.map(Path::to_path_buf)
.unwrap_or_else(default_output_root);
fs::create_dir_all(&output_root)?;
let run_dir = create_timestamped_run_dir(&output_root, "secom")?;
let non_intrusive_artifacts = materialize_non_intrusive_artifacts(&run_dir)?;
write_readme_first(&run_dir, &config)?;
write_operator_summary(&run_dir, &config)?;
write_json_pretty(&run_dir.join("dataset_summary.json"), &prepared.summary)?;
write_json_pretty(&run_dir.join("parameter_manifest.json"), &config)?;
write_json_pretty(
&run_dir.join("run_configuration.json"),
&RunConfiguration {
dataset: "SECOM".into(),
config: config.clone(),
data_root: data_root.display().to_string(),
output_root: output_root.display().to_string(),
secom_fetch_if_missing: fetch_if_missing,
},
)?;
write_json_pretty(&run_dir.join("benchmark_metrics.json"), &metrics)?;
write_json_pretty(
&run_dir.join("secom_archive_layout.json"),
&secom_archive_layout,
)?;
write_json_pretty(
&run_dir.join("phm2018_support_status.json"),
&phm_support_status(data_root),
)?;
write_json_pretty(&run_dir.join("heuristics_bank.json"), &heuristics)?;
write_json_pretty(
&run_dir.join("baseline_comparison_summary.json"),
&build_baseline_comparison_summary(&metrics, &dsa, &secom_archive_layout, &config),
)?;
write_json_pretty(
&run_dir.join("dsa_vs_baselines.json"),
&dsa.comparison_summary,
)?;
write_json_pretty(
&run_dir.join("dsa_parameter_manifest.json"),
&dsa.parameter_manifest,
)?;
write_json_pretty(&run_dir.join("dsa_grid_summary.json"), &dsa_grid_summary)?;
write_feature_ranking_csv(&run_dir.join("dsa_feature_ranking.csv"), &feature_ranking)?;
write_feature_ranking_csv(
&run_dir.join("dsa_feature_ranking_recall_aware.csv"),
&optimization.recall_aware_feature_ranking,
)?;
write_feature_ranking_csv(
&run_dir.join("dsa_feature_ranking_dsfb_aware.csv"),
&optimization.dsfb_aware_feature_ranking,
)?;
write_feature_ranking_csv(
&run_dir.join("dsa_feature_ranking_burden_aware.csv"),
&optimization.burden_aware_feature_ranking,
)?;
write_feature_ranking_comparison_csv(
&run_dir.join("dsa_feature_ranking_comparison.csv"),
&optimization.ranking_comparison,
)?;
write_json_pretty(
&run_dir.join("dsa_seed_feature_check.json"),
&seed_feature_check,
)?;
write_json_pretty(&run_dir.join("dsa_feature_cohorts.json"), &feature_cohorts)?;
write_json_pretty(
&run_dir.join("dsa_feature_policy_overrides.json"),
&optimization.feature_policy_overrides,
)?;
write_feature_policy_summary_csv(
&run_dir.join("dsa_feature_policy_summary.csv"),
&optimization.feature_policy_summary,
)?;
write_recall_rescue_results_csv(
&run_dir.join("dsa_recall_rescue_results.csv"),
&optimization.recall_rescue_results,
)?;
write_recall_critical_features_csv(
&run_dir.join("dsa_recall_critical_features.csv"),
&optimization.recall_critical_features,
)?;
write_cohort_results_csv(
&run_dir.join("dsa_pareto_frontier.csv"),
&optimization.pareto_frontier,
)?;
write_cohort_results_csv(
&run_dir.join("dsa_stage_a_candidates.csv"),
&optimization.stage_a_candidates,
)?;
write_cohort_results_csv(
&run_dir.join("dsa_stage_b_candidates.csv"),
&optimization.stage_b_candidates,
)?;
write_missed_failure_diagnostics_csv(
&run_dir.join("dsa_missed_failure_diagnostics.csv"),
&optimization.missed_failure_diagnostics,
)?;
write_cohort_results_csv(
&run_dir.join("dsa_cohort_results.csv"),
&cohort_summary.cohort_results,
)?;
write_cohort_results_csv(
&run_dir.join("dsa_grid_results.csv"),
&cohort_summary.cohort_results,
)?;
write_cohort_results_csv(
&run_dir.join("dsa_cohort_results_recall_aware.csv"),
&optimization.recall_aware_execution.summary.cohort_results,
)?;
write_cohort_results_csv(
&run_dir.join("dsa_cohort_results_dsfb_aware.csv"),
&optimization.dsfb_aware_execution.summary.cohort_results,
)?;
write_cohort_results_csv(
&run_dir.join("dsa_cohort_results_burden_aware.csv"),
&optimization.burden_aware_execution.summary.cohort_results,
)?;
write_json_pretty(&run_dir.join("dsa_cohort_summary.json"), &cohort_summary)?;
write_json_pretty(
&run_dir.join("dsa_cohort_summary_recall_aware.json"),
&optimization.recall_aware_execution.summary,
)?;
write_json_pretty(
&run_dir.join("dsa_cohort_summary_dsfb_aware.json"),
&optimization.dsfb_aware_execution.summary,
)?;
write_json_pretty(
&run_dir.join("dsa_cohort_summary_burden_aware.json"),
&optimization.burden_aware_execution.summary,
)?;
write_precursor_quality_csv(
&run_dir.join("dsa_cohort_precursor_quality.csv"),
&cohort_summary.cohort_results,
)?;
write_motif_policy_contributions_csv(
&run_dir.join("dsa_motif_policy_contributions.csv"),
&optimization.optimized_execution.motif_policy_contributions,
)?;
write_policy_contribution_analysis_csv(
&run_dir.join("dsa_policy_contribution_analysis.csv"),
&optimization.policy_contribution_analysis,
)?;
write_json_pretty(
&run_dir.join("dsa_rating_delta_forecast.json"),
&rating_delta_forecast,
)?;
write_json_pretty(
&run_dir.join("dsa_delta_target_assessment.json"),
&optimization.delta_target_assessment,
)?;
write_json_pretty(
&run_dir.join("dsa_operator_baselines.json"),
&optimization.operator_baselines,
)?;
write_json_pretty(
&run_dir.join("dsa_operator_delta_targets.json"),
&optimization.operator_delta_targets,
)?;
write_operator_delta_attainment_matrix_csv(
&run_dir.join("dsa_operator_delta_attainment_matrix.csv"),
&optimization.operator_delta_attainment_matrix,
)?;
write_operator_burden_contributions_csv(
&run_dir.join("dsa_policy_operator_burden_contributions.csv"),
&optimization.policy_operator_burden_contributions,
)?;
write_recall_recovery_efficiency_csv(
&run_dir.join("dsa_recall_recovery_efficiency.csv"),
&optimization.recall_recovery_efficiency,
)?;
write_single_change_iteration_log_csv(
&run_dir.join("dsfb_single_change_iteration_log.csv"),
&optimization.single_change_iteration_log,
)?;
write_json_pretty(
&run_dir.join("optimization_log.json"),
&optimization.single_change_iteration_log,
)?;
write_json_pretty(
&run_dir.join("failures_index.json"),
&failure_driven.failures_index,
)?;
write_serialized_csv(
&run_dir.join("missed_failure_priority.csv"),
&failure_driven.missed_failure_priority,
)?;
for failure_case in &failure_driven.failure_cases {
write_json_pretty(
&run_dir.join(format!("failure_case_{}.json", failure_case.failure_id)),
failure_case,
)?;
}
write_json_pretty(
&run_dir.join("feature_motif_grounding.json"),
&failure_driven.feature_motif_grounding,
)?;
write_json_pretty(
&run_dir.join("feature_to_motif.json"),
&failure_driven.feature_to_motif,
)?;
write_json_pretty(
&run_dir.join("negative_control_report.json"),
&failure_driven.negative_control_report,
)?;
write_json_pretty(
&run_dir.join("dsfb_heuristics_bank_minimal.json"),
&failure_driven.minimal_heuristics_bank,
)?;
write_serialized_csv(
&run_dir.join("dsfb_heuristic_provenance.csv"),
&failure_driven.heuristic_provenance,
)?;
write_serialized_csv(
&run_dir.join("policy_burden_summary.csv"),
&failure_driven.policy_burden_summary,
)?;
write_serialized_csv(
&run_dir.join("dsfb_feature_role_validation.csv"),
&failure_driven.feature_role_validation,
)?;
write_serialized_csv(
&run_dir.join("dsfb_group_validation.csv"),
&failure_driven.group_validation,
)?;
for case in &failure_driven.dsfb_vs_ewma_cases {
write_json_pretty(
&run_dir.join(format!("dsfb_vs_ewma_case_{}.json", case.failure_id)),
case,
)?;
}
write_cohort_results_csv(
&run_dir.join("dsa_stage1_candidates.csv"),
&optimization.stage1_candidates,
)?;
write_cohort_results_csv(
&run_dir.join("dsa_stage2_candidates.csv"),
&optimization.stage2_candidates,
)?;
if let Some(analysis) = &cohort_summary.failure_analysis {
write_cohort_failure_analysis_md(
&run_dir.join("dsa_cohort_failure_analysis.md"),
analysis,
)?;
write_heuristic_policy_failure_analysis_md(
&run_dir.join("dsa_heuristic_policy_failure_analysis.md"),
analysis,
)?;
}
if let Some(analysis) = &rating_delta_failure_analysis {
crate::cohort::write_rating_failure_analysis_md(
&run_dir.join("dsa_rating_delta_failure_analysis.md"),
analysis,
)?;
}
write_feature_metrics_csv(&run_dir.join("feature_metrics.csv"), &metrics)?;
write_per_failure_run_signals_csv(
&run_dir.join("per_failure_run_signals.csv"),
&metrics.per_failure_run_signals,
)?;
write_dsa_metrics_csv(&run_dir.join("dsa_metrics.csv"), &prepared, &nominal, &dsa)?;
write_dsa_run_signals_csv(
&run_dir.join("dsa_run_signals.csv"),
&prepared,
&dsa.run_signals,
)?;
write_per_failure_run_dsa_signals_csv(
&run_dir.join("per_failure_run_dsa_signals.csv"),
&dsa.per_failure_run_signals,
)?;
write_lead_time_metrics_csv(
&run_dir.join("lead_time_metrics.csv"),
&metrics.per_failure_run_signals,
)?;
write_density_metrics_csv(
&run_dir.join("density_metrics.csv"),
&metrics.density_metrics,
)?;
write_trace_csvs(
&run_dir, &prepared, &residuals, &signs, &baselines, &grammar,
)?;
write_dsfb_signs_csv(
&run_dir.join("dsfb_signs.csv"),
&prepared,
&residuals,
&signs,
)?;
write_serialized_csv(
&run_dir.join("dsfb_feature_signs.csv"),
&scaffold_semiotics.feature_signs,
)?;
write_dsfb_motifs_csv(&run_dir.join("dsfb_motifs.csv"), &motifs)?;
write_dsfb_motif_labels_per_time_csv(
&run_dir.join("dsfb_motif_labels_per_time.csv"),
&prepared,
&motifs,
)?;
write_serialized_csv(
&run_dir.join("dsfb_feature_motif_timeline.csv"),
&scaffold_semiotics.feature_motif_timeline,
)?;
write_serialized_csv(
&run_dir.join("feature_motif_timeline.csv"),
&scaffold_semiotics.feature_motif_timeline,
)?;
write_dsfb_grammar_states_csv(
&run_dir.join("dsfb_grammar_states.csv"),
&prepared,
&grammar,
)?;
write_serialized_csv(
&run_dir.join("dsfb_feature_grammar_states.csv"),
&scaffold_semiotics.feature_grammar_states,
)?;
write_serialized_csv(
&run_dir.join("dsfb_envelope_interaction_summary.csv"),
&scaffold_semiotics.envelope_interaction_summary,
)?;
write_json_pretty(
&run_dir.join("dsfb_heuristics_bank_expanded.json"),
&scaffold_semiotics.heuristics_bank_expanded,
)?;
write_dsfb_semantic_matches_csv(
&run_dir.join("dsfb_semantic_matches.csv"),
&semantic_layer.semantic_matches,
)?;
write_dsfb_semantic_matches_csv(
&run_dir.join("dsfb_semantic_ranked_candidates.csv"),
&semantic_layer.ranked_candidates,
)?;
write_serialized_csv(
&run_dir.join("dsfb_feature_policy_decisions.csv"),
&scaffold_semiotics.feature_policy_decisions,
)?;
write_serialized_csv(
&run_dir.join("policy_decisions.csv"),
&scaffold_semiotics.feature_policy_decisions,
)?;
let traceability_entries = build_traceability_entries(&scaffold_semiotics);
write_traceability_json(
&run_dir.join("dsfb_traceability.json"),
&traceability_entries,
)?;
{
let manifest = DsfbRunManifest::new(
Utc::now().to_rfc3339(),
"batch_secom_no_recipe_context".to_string(),
traceability_entries.len(),
);
manifest.write(&run_dir.join("dsfb_run_manifest.json"))?;
}
write_json_pretty(
&run_dir.join("dsfb_group_definitions.json"),
&scaffold_semiotics.group_definitions,
)?;
write_serialized_csv(
&run_dir.join("dsfb_group_signs.csv"),
&scaffold_semiotics.group_signs,
)?;
write_serialized_csv(
&run_dir.join("dsfb_group_grammar_states.csv"),
&scaffold_semiotics.group_grammar_states,
)?;
write_serialized_csv(
&run_dir.join("dsfb_group_semantic_matches.csv"),
&scaffold_semiotics.group_semantic_matches,
)?;
write_json_pretty(
&run_dir.join("dsfb_structural_delta_metrics.json"),
&semantic_layer.structural_delta_metrics,
)?;
write_json_pretty(
&run_dir.join("recurrent_boundary_stats.json"),
&secom_addendum.recurrent_boundary_stats,
)?;
write_serialized_csv(
&run_dir.join("recurrent_boundary_tradeoff_curve.csv"),
&secom_addendum.recurrent_boundary_tradeoff_curve,
)?;
draw_recurrent_boundary_tradeoff_plot(
&run_dir.join("recurrent_boundary_tradeoff_plot.png"),
&secom_addendum.recurrent_boundary_tradeoff_curve,
)?;
write_serialized_csv(
&run_dir.join("dsfb_metric_regrounding.csv"),
&secom_addendum.metric_regrounding,
)?;
write_json_pretty(
&run_dir.join("target_d_regression_analysis.json"),
&secom_addendum.target_d_regression_analysis,
)?;
write_json_pretty(
&run_dir.join("missed_failure_root_cause.json"),
&secom_addendum.missed_failure_root_cause,
)?;
write_serialized_csv(
&run_dir.join("lead_time_comparison.csv"),
&secom_addendum.lead_time_comparison,
)?;
write_json_pretty(
&run_dir.join("lead_time_explanation.json"),
&secom_addendum.lead_time_explanation,
)?;
write_json_pretty(
&run_dir.join("episode_precision_metrics.json"),
&secom_addendum.episode_precision_metrics,
)?;
write_serialized_csv(
&run_dir.join("dsfb_episode_summary.csv"),
&[DsfbEpisodeSummaryRow {
raw_boundary_episodes: optimization.operator_delta_targets.baseline_episode_count,
dsfb_episodes: optimization.operator_delta_targets.optimized_episode_count,
episode_collapse_fraction: optimization.operator_delta_targets.delta_episode_count,
}],
)?;
write_serialized_csv(
&run_dir.join("dsfb_episode_precision.csv"),
&[DsfbEpisodePrecisionRow {
raw_boundary_precision: secom_addendum.episode_precision_metrics.raw_alarm_precision,
dsfb_precision: secom_addendum.episode_precision_metrics.dsfb_precision,
precision_gain_factor: secom_addendum
.episode_precision_metrics
.precision_gain_factor,
raw_boundary_episodes: secom_addendum.episode_precision_metrics.raw_alarm_count,
dsfb_episodes: secom_addendum.episode_precision_metrics.dsfb_episode_count,
dsfb_pre_failure_episodes: secom_addendum
.episode_precision_metrics
.dsfb_pre_failure_episode_count,
}],
)?;
write_serialized_csv(
&run_dir.join("dsfb_recall_metrics.csv"),
&[DsfbRecallMetricsRow {
detected_failures: optimization
.operator_delta_targets
.selected_configuration
.failure_recall,
total_failures: optimization
.operator_delta_targets
.selected_configuration
.failure_runs,
recall: optimization
.operator_delta_targets
.selected_configuration
.failure_recall as f64
/ optimization
.operator_delta_targets
.selected_configuration
.failure_runs
.max(1) as f64,
}],
)?;
fs::write(
run_dir.join("paper_abstract_artifact.txt"),
&secom_addendum.paper_abstract_artifact,
)?;
let mut figures = generate_figures(
&run_dir, &prepared, &nominal, &residuals, &signs, &baselines, &grammar, &metrics, &dsa,
&config,
)?;
figures
.files
.push("dsfb_non_intrusive_architecture.png".into());
let report = write_reports(
&run_dir,
&config,
&metrics,
&dsa,
&optimization,
&optimization.delta_target_assessment,
&failure_driven,
&feature_cohorts,
&cohort_summary,
&rating_delta_forecast,
&secom_addendum,
&figures,
&heuristics,
&phm_support_status(data_root),
&secom_archive_layout,
)?;
let manifest_path = run_dir.join("artifact_manifest.json");
let metrics_path = run_dir.join("benchmark_metrics.json");
let phm2018_status = phm_support_status(data_root);
let zip_path = run_dir.join("run_bundle.zip");
let report_pdf_alias_path = run_dir.join("report.pdf");
let results_zip_path = run_dir.join("results.zip");
write_json_pretty(
&manifest_path,
&ArtifactManifest {
dataset: "SECOM".into(),
run_dir: run_dir.display().to_string(),
metrics_summary_path: metrics_path.display().to_string(),
baseline_comparison_summary_path: run_dir
.join("baseline_comparison_summary.json")
.display()
.to_string(),
dsa_vs_baselines_summary_path: run_dir
.join("dsa_vs_baselines.json")
.display()
.to_string(),
dsa_parameter_manifest_path: run_dir
.join("dsa_parameter_manifest.json")
.display()
.to_string(),
dsa_grid_results_path: run_dir.join("dsa_grid_results.csv").display().to_string(),
dsa_grid_summary_path: run_dir.join("dsa_grid_summary.json").display().to_string(),
dsa_feature_ranking_path: run_dir
.join("dsa_feature_ranking.csv")
.display()
.to_string(),
dsa_feature_ranking_recall_aware_path: run_dir
.join("dsa_feature_ranking_recall_aware.csv")
.display()
.to_string(),
dsa_feature_ranking_dsfb_aware_path: run_dir
.join("dsa_feature_ranking_dsfb_aware.csv")
.display()
.to_string(),
dsa_feature_ranking_burden_aware_path: run_dir
.join("dsa_feature_ranking_burden_aware.csv")
.display()
.to_string(),
dsa_feature_ranking_comparison_path: run_dir
.join("dsa_feature_ranking_comparison.csv")
.display()
.to_string(),
dsa_seed_feature_check_path: run_dir
.join("dsa_seed_feature_check.json")
.display()
.to_string(),
dsa_feature_cohorts_path: run_dir
.join("dsa_feature_cohorts.json")
.display()
.to_string(),
dsa_feature_policy_overrides_path: run_dir
.join("dsa_feature_policy_overrides.json")
.display()
.to_string(),
dsa_feature_policy_summary_path: run_dir
.join("dsa_feature_policy_summary.csv")
.display()
.to_string(),
dsa_recall_rescue_results_path: run_dir
.join("dsa_recall_rescue_results.csv")
.display()
.to_string(),
dsa_recall_critical_features_path: run_dir
.join("dsa_recall_critical_features.csv")
.display()
.to_string(),
dsa_pareto_frontier_path: run_dir
.join("dsa_pareto_frontier.csv")
.display()
.to_string(),
dsa_stage_a_candidates_path: run_dir
.join("dsa_stage_a_candidates.csv")
.display()
.to_string(),
dsa_stage_b_candidates_path: run_dir
.join("dsa_stage_b_candidates.csv")
.display()
.to_string(),
dsa_stage1_candidates_path: run_dir
.join("dsa_stage1_candidates.csv")
.display()
.to_string(),
dsa_stage2_candidates_path: run_dir
.join("dsa_stage2_candidates.csv")
.display()
.to_string(),
dsa_missed_failure_diagnostics_path: run_dir
.join("dsa_missed_failure_diagnostics.csv")
.display()
.to_string(),
dsa_delta_target_assessment_path: run_dir
.join("dsa_delta_target_assessment.json")
.display()
.to_string(),
dsa_operator_baselines_path: run_dir
.join("dsa_operator_baselines.json")
.display()
.to_string(),
dsa_operator_delta_targets_path: run_dir
.join("dsa_operator_delta_targets.json")
.display()
.to_string(),
dsa_operator_delta_attainment_matrix_path: run_dir
.join("dsa_operator_delta_attainment_matrix.csv")
.display()
.to_string(),
dsa_policy_operator_burden_contributions_path: run_dir
.join("dsa_policy_operator_burden_contributions.csv")
.display()
.to_string(),
dsa_recall_recovery_efficiency_path: run_dir
.join("dsa_recall_recovery_efficiency.csv")
.display()
.to_string(),
dsfb_single_change_iteration_log_path: run_dir
.join("dsfb_single_change_iteration_log.csv")
.display()
.to_string(),
optimization_log_path: run_dir.join("optimization_log.json").display().to_string(),
failures_index_path: run_dir.join("failures_index.json").display().to_string(),
missed_failure_priority_path: run_dir
.join("missed_failure_priority.csv")
.display()
.to_string(),
failure_case_paths: failure_driven
.failure_cases
.iter()
.map(|case| {
run_dir
.join(format!("failure_case_{}.json", case.failure_id))
.display()
.to_string()
})
.collect(),
feature_motif_grounding_path: run_dir
.join("feature_motif_grounding.json")
.display()
.to_string(),
feature_to_motif_path: run_dir.join("feature_to_motif.json").display().to_string(),
negative_control_report_path: run_dir
.join("negative_control_report.json")
.display()
.to_string(),
dsfb_heuristics_bank_minimal_path: run_dir
.join("dsfb_heuristics_bank_minimal.json")
.display()
.to_string(),
dsfb_heuristic_provenance_path: run_dir
.join("dsfb_heuristic_provenance.csv")
.display()
.to_string(),
policy_decisions_path: run_dir.join("policy_decisions.csv").display().to_string(),
policy_burden_summary_path: run_dir
.join("policy_burden_summary.csv")
.display()
.to_string(),
dsfb_feature_role_validation_path: run_dir
.join("dsfb_feature_role_validation.csv")
.display()
.to_string(),
dsfb_group_validation_path: run_dir
.join("dsfb_group_validation.csv")
.display()
.to_string(),
dsfb_vs_ewma_case_paths: failure_driven
.dsfb_vs_ewma_cases
.iter()
.map(|case| {
run_dir
.join(format!("dsfb_vs_ewma_case_{}.json", case.failure_id))
.display()
.to_string()
})
.collect(),
dsa_cohort_results_path: run_dir.join("dsa_cohort_results.csv").display().to_string(),
dsa_cohort_results_recall_aware_path: run_dir
.join("dsa_cohort_results_recall_aware.csv")
.display()
.to_string(),
dsa_cohort_results_dsfb_aware_path: run_dir
.join("dsa_cohort_results_dsfb_aware.csv")
.display()
.to_string(),
dsa_cohort_results_burden_aware_path: run_dir
.join("dsa_cohort_results_burden_aware.csv")
.display()
.to_string(),
dsa_cohort_summary_path: run_dir
.join("dsa_cohort_summary.json")
.display()
.to_string(),
dsa_cohort_summary_recall_aware_path: run_dir
.join("dsa_cohort_summary_recall_aware.json")
.display()
.to_string(),
dsa_cohort_summary_dsfb_aware_path: run_dir
.join("dsa_cohort_summary_dsfb_aware.json")
.display()
.to_string(),
dsa_cohort_summary_burden_aware_path: run_dir
.join("dsa_cohort_summary_burden_aware.json")
.display()
.to_string(),
dsa_cohort_precursor_quality_path: run_dir
.join("dsa_cohort_precursor_quality.csv")
.display()
.to_string(),
dsa_cohort_failure_analysis_path: cohort_summary.failure_analysis.as_ref().map(|_| {
run_dir
.join("dsa_cohort_failure_analysis.md")
.display()
.to_string()
}),
dsa_heuristic_policy_failure_analysis_path: cohort_summary
.failure_analysis
.as_ref()
.map(|_| {
run_dir
.join("dsa_heuristic_policy_failure_analysis.md")
.display()
.to_string()
}),
dsa_motif_policy_contributions_path: run_dir
.join("dsa_motif_policy_contributions.csv")
.display()
.to_string(),
dsa_policy_contribution_analysis_path: run_dir
.join("dsa_policy_contribution_analysis.csv")
.display()
.to_string(),
dsa_rating_delta_forecast_path: run_dir
.join("dsa_rating_delta_forecast.json")
.display()
.to_string(),
dsa_rating_delta_failure_analysis_path: rating_delta_failure_analysis.as_ref().map(
|_| {
run_dir
.join("dsa_rating_delta_failure_analysis.md")
.display()
.to_string()
},
),
lead_time_metrics_path: run_dir.join("lead_time_metrics.csv").display().to_string(),
density_metrics_path: run_dir.join("density_metrics.csv").display().to_string(),
cusum_baseline_path: run_dir.join("cusum_baseline.csv").display().to_string(),
run_energy_baseline_path: run_dir
.join("run_energy_baseline.csv")
.display()
.to_string(),
pca_fdc_baseline_path: run_dir.join("pca_fdc_baseline.csv").display().to_string(),
per_failure_run_signals_path: run_dir
.join("per_failure_run_signals.csv")
.display()
.to_string(),
dsa_metrics_path: run_dir.join("dsa_metrics.csv").display().to_string(),
dsa_run_signals_path: run_dir.join("dsa_run_signals.csv").display().to_string(),
per_failure_run_dsa_signals_path: run_dir
.join("per_failure_run_dsa_signals.csv")
.display()
.to_string(),
dsfb_signs_path: run_dir.join("dsfb_signs.csv").display().to_string(),
dsfb_feature_signs_path: run_dir.join("dsfb_feature_signs.csv").display().to_string(),
dsfb_motifs_path: run_dir.join("dsfb_motifs.csv").display().to_string(),
dsfb_motif_labels_per_time_path: run_dir
.join("dsfb_motif_labels_per_time.csv")
.display()
.to_string(),
dsfb_feature_motif_timeline_path: run_dir
.join("dsfb_feature_motif_timeline.csv")
.display()
.to_string(),
dsfb_grammar_states_path: run_dir
.join("dsfb_grammar_states.csv")
.display()
.to_string(),
dsfb_feature_grammar_states_path: run_dir
.join("dsfb_feature_grammar_states.csv")
.display()
.to_string(),
dsfb_envelope_interaction_summary_path: run_dir
.join("dsfb_envelope_interaction_summary.csv")
.display()
.to_string(),
dsfb_heuristics_bank_expanded_path: run_dir
.join("dsfb_heuristics_bank_expanded.json")
.display()
.to_string(),
dsfb_semantic_matches_path: run_dir
.join("dsfb_semantic_matches.csv")
.display()
.to_string(),
dsfb_semantic_ranked_candidates_path: run_dir
.join("dsfb_semantic_ranked_candidates.csv")
.display()
.to_string(),
dsfb_feature_policy_decisions_path: run_dir
.join("dsfb_feature_policy_decisions.csv")
.display()
.to_string(),
dsfb_traceability_path: run_dir
.join("dsfb_traceability.json")
.display()
.to_string(),
dsfb_group_definitions_path: run_dir
.join("dsfb_group_definitions.json")
.display()
.to_string(),
dsfb_group_signs_path: run_dir.join("dsfb_group_signs.csv").display().to_string(),
dsfb_group_grammar_states_path: run_dir
.join("dsfb_group_grammar_states.csv")
.display()
.to_string(),
dsfb_group_semantic_matches_path: run_dir
.join("dsfb_group_semantic_matches.csv")
.display()
.to_string(),
dsfb_structural_delta_metrics_path: run_dir
.join("dsfb_structural_delta_metrics.json")
.display()
.to_string(),
recurrent_boundary_stats_path: run_dir
.join("recurrent_boundary_stats.json")
.display()
.to_string(),
recurrent_boundary_tradeoff_curve_path: run_dir
.join("recurrent_boundary_tradeoff_curve.csv")
.display()
.to_string(),
recurrent_boundary_tradeoff_plot_path: run_dir
.join("recurrent_boundary_tradeoff_plot.png")
.display()
.to_string(),
dsfb_metric_regrounding_path: run_dir
.join("dsfb_metric_regrounding.csv")
.display()
.to_string(),
target_d_regression_analysis_path: run_dir
.join("target_d_regression_analysis.json")
.display()
.to_string(),
missed_failure_root_cause_path: run_dir
.join("missed_failure_root_cause.json")
.display()
.to_string(),
lead_time_comparison_path: run_dir
.join("lead_time_comparison.csv")
.display()
.to_string(),
lead_time_explanation_path: run_dir
.join("lead_time_explanation.json")
.display()
.to_string(),
episode_precision_metrics_path: run_dir
.join("episode_precision_metrics.json")
.display()
.to_string(),
dsfb_episode_summary_path: run_dir
.join("dsfb_episode_summary.csv")
.display()
.to_string(),
dsfb_episode_precision_path: run_dir
.join("dsfb_episode_precision.csv")
.display()
.to_string(),
dsfb_recall_metrics_path: run_dir
.join("dsfb_recall_metrics.csv")
.display()
.to_string(),
paper_abstract_artifact_path: run_dir
.join("paper_abstract_artifact.txt")
.display()
.to_string(),
secom_archive_layout_path: run_dir
.join("secom_archive_layout.json")
.display()
.to_string(),
drsc_trace_path: figures
.drsc
.as_ref()
.map(|drsc| run_dir.join(&drsc.trace_csv).display().to_string()),
drsc_figure_path: figures.drsc.as_ref().map(|drsc| {
run_dir
.join("figures")
.join(&drsc.figure_file)
.display()
.to_string()
}),
drsc_dsa_combined_trace_path: figures
.drsc_dsa_combined
.as_ref()
.map(|combined| run_dir.join(&combined.trace_csv).display().to_string()),
drsc_dsa_combined_figure_path: figures.drsc_dsa_combined.as_ref().map(|combined| {
run_dir
.join("figures")
.join(&combined.figure_file)
.display()
.to_string()
}),
dsa_focus_trace_path: figures
.dsa_focus
.as_ref()
.map(|dsa_focus| run_dir.join(&dsa_focus.trace_csv).display().to_string()),
dsa_focus_figure_path: figures.dsa_focus.as_ref().map(|dsa_focus| {
run_dir
.join("figures")
.join(&dsa_focus.figure_file)
.display()
.to_string()
}),
non_intrusive_interface_spec_path: non_intrusive_artifacts
.interface_spec_path
.display()
.to_string(),
non_intrusive_architecture_png_path: non_intrusive_artifacts
.architecture_png_path
.display()
.to_string(),
non_intrusive_architecture_svg_path: non_intrusive_artifacts
.architecture_svg_path
.display()
.to_string(),
report_markdown_path: report.markdown_path.display().to_string(),
report_tex_path: report.tex_path.display().to_string(),
report_pdf_path: report
.pdf_path
.as_ref()
.map(|path| path.display().to_string()),
report_pdf_alias_path: report
.pdf_path
.as_ref()
.map(|_| report_pdf_alias_path.display().to_string()),
zip_path: zip_path.display().to_string(),
results_zip_path: results_zip_path.display().to_string(),
},
)?;
if let Some(pdf_path) = &report.pdf_path {
fs::copy(pdf_path, &report_pdf_alias_path)?;
}
zip_directory(&run_dir, &zip_path)?;
fs::copy(&zip_path, &results_zip_path)?;
Ok(SecomRunArtifacts {
run_dir,
report,
figures,
metrics_path,
manifest_path,
zip_path,
phm2018_status,
paper_lock_metrics,
})
}
fn write_readme_first(run_dir: &std::path::Path, config: &crate::config::PipelineConfig) -> crate::error::Result<()> {
let content = format!(
r#"DSFB-SEMICONDUCTOR — RUN SUMMARY
=================================
CONFIGURATION
W (drift_window) : {drift_window}
K (pre_failure_lookback_runs) : {lookback}
tau (dsa.alert_tau) : {tau:.1}
m (corroborating_feature_min) : {m}
envelope_sigma : {sigma:.1}
strategy : all_features [compression_biased]
WHAT DSFB DOES
- Reads residual streams produced by SECOM feature extraction (read-only).
- Restructures those residuals into a compact set of structured episodes
annotated with grammar states (Admissible / Boundary / Violation).
- Emits advisory policy decisions (Silent / Review / Escalate) per feature.
- Writes no data back to any upstream system.
WHAT DSFB DOES NOT DO (explicit non-claims)
- Does NOT alter any FDC, SPC, EWMA, or CUSUM system.
- Does NOT predict failure lead-time or claim empirical lead-time advantage.
- Does NOT replace process engineering review or control plan sign-off.
- Does NOT claim physical attribution for any motif class observed in SECOM.
- Is NOT a certified or production-qualified fault-detection system.
- Removal of the DSFB layer leaves all upstream systems entirely unchanged.
FILES IN THIS DIRECTORY
metrics_summary.json — Full benchmark metric tree
baseline_comparison_summary.json — DSFB vs. EWMA / CUSUM / PCA baselines
dsfb_episode_precision.csv — Filtered episode count and precision
dsfb_recall_metrics.csv — Failure-run recall
report.tex / report.pdf — LaTeX narrative report
dsfb_semiconductor_secom.zip — Full reproducibility archive
REPRODUCIBILITY
Re-run with: cargo run --release -- run-secom
Paper-lock: cargo run --release -- paper-lock
All outputs are deterministic under fixed config and dataset.
"#,
drift_window = config.drift_window,
lookback = config.pre_failure_lookback_runs,
tau = config.dsa.alert_tau,
m = config.dsa.corroborating_feature_count_min,
sigma = config.envelope_sigma,
);
fs::write(run_dir.join("README_FIRST.txt"), content).map_err(Into::into)
}
fn write_operator_summary(
run_dir: &std::path::Path,
config: &crate::config::PipelineConfig,
) -> crate::error::Result<()> {
let content = format!(
r#"DSFB-SEMICONDUCTOR — OPERATOR RUN SUMMARY
==========================================
PURPOSE
This run applies the DSFB structural semiotics observer to SECOM residuals.
The observer is read-only: it does not modify any upstream control system.
SELECTED CONFIGURATION
DSA drift window : {dsa_window} (W in paper)
DSA persistence runs : {dsa_persistence} (K in paper)
DSA alert threshold : {dsa_tau:.1} (tau in paper)
DSA corroboration count : {dsa_m} (m in paper)
Feature strategy : all_features [compression_biased]
Grammar drift window : {drift_window}
Envelope sigma : {sigma:.1}
WHAT THIS RUN PRODUCES
- Structured episode list (Watch / Review / Escalate per feature)
- Traceability chain: dsfb_traceability.json
- Benchmark metrics: benchmark_metrics.json
- Full report: report.tex / report.pdf
WHAT THIS RUN DOES NOT PRODUCE
- No modifications to SPC, EWMA, FDC, or CUSUM thresholds
- No write-back to upstream control systems
- No physical attribution claims
- No predictive lead-time guarantee
REPRODUCTION
cargo run --release --bin dsfb-semiconductor -- run-secom
cargo run --release --bin dsfb-semiconductor -- paper-lock
All outputs are deterministic under fixed configuration and dataset.
"#,
dsa_window = config.dsa.window,
dsa_persistence = config.dsa.persistence_runs,
dsa_tau = config.dsa.alert_tau,
dsa_m = config.dsa.corroborating_feature_count_min,
drift_window = config.drift_window,
sigma = config.envelope_sigma,
);
fs::write(run_dir.join("RUN_SUMMARY_OPERATOR.txt"), content).map_err(Into::into)
}
fn build_baseline_comparison_summary(
metrics: &BenchmarkMetrics,
dsa: &DsaEvaluation,
secom_archive_layout: &SecomArchiveLayout,
config: &PipelineConfig,
) -> BaselineComparisonSummary {
BaselineComparisonSummary {
dataset: "SECOM".into(),
secom_archive_layout_note: secom_archive_layout.note.clone(),
feature_count_used_by_crate: metrics.summary.dataset_summary.feature_count,
failure_runs: metrics.summary.failure_runs,
analyzable_feature_count: metrics.summary.analyzable_feature_count,
grammar_imputation_suppression_points: metrics
.summary
.grammar_imputation_suppression_points,
lookback_runs: config.pre_failure_lookback_runs,
failure_run_recall: FailureRunRecallSummary {
dsfb_raw_signal: metrics.summary.failure_runs_with_preceding_dsfb_raw_signal,
dsfb_persistent_signal: metrics
.summary
.failure_runs_with_preceding_dsfb_persistent_signal,
dsfb_raw_boundary_signal: metrics
.summary
.failure_runs_with_preceding_dsfb_raw_boundary_signal,
dsfb_persistent_boundary_signal: metrics
.summary
.failure_runs_with_preceding_dsfb_persistent_boundary_signal,
dsfb_raw_violation_signal: metrics
.summary
.failure_runs_with_preceding_dsfb_raw_violation_signal,
dsfb_persistent_violation_signal: metrics
.summary
.failure_runs_with_preceding_dsfb_persistent_violation_signal,
dsfb_dsa_signal: dsa.summary.failure_run_recall,
ewma_signal: metrics.summary.failure_runs_with_preceding_ewma_signal,
cusum_signal: metrics.summary.failure_runs_with_preceding_cusum_signal,
run_energy_signal: metrics
.summary
.failure_runs_with_preceding_run_energy_signal,
pca_fdc_signal: metrics.summary.failure_runs_with_preceding_pca_fdc_signal,
threshold_signal: metrics.summary.failure_runs_with_preceding_threshold_signal,
},
pass_run_nuisance_proxy: PassRunNuisanceSummary {
dsfb_raw_boundary_signal_runs: metrics.summary.pass_runs_with_dsfb_raw_boundary_signal,
dsfb_persistent_boundary_signal_runs: metrics
.summary
.pass_runs_with_dsfb_persistent_boundary_signal,
dsfb_raw_violation_signal_runs: metrics
.summary
.pass_runs_with_dsfb_raw_violation_signal,
dsfb_persistent_violation_signal_runs: metrics
.summary
.pass_runs_with_dsfb_persistent_violation_signal,
dsfb_dsa_signal_runs: (dsa.summary.pass_run_nuisance_proxy
* metrics.summary.pass_runs as f64)
.round() as usize,
ewma_signal_runs: metrics.summary.pass_runs_with_ewma_signal,
cusum_signal_runs: metrics.summary.pass_runs_with_cusum_signal,
run_energy_signal_runs: metrics.summary.pass_runs_with_run_energy_signal,
pca_fdc_signal_runs: metrics.summary.pass_runs_with_pca_fdc_signal,
threshold_signal_runs: metrics.summary.pass_runs_with_threshold_signal,
dsfb_raw_boundary_signal_rate: metrics.summary.pass_run_dsfb_raw_boundary_nuisance_rate,
dsfb_persistent_boundary_signal_rate: metrics
.summary
.pass_run_dsfb_persistent_boundary_nuisance_rate,
dsfb_raw_violation_signal_rate: metrics
.summary
.pass_run_dsfb_raw_violation_nuisance_rate,
dsfb_persistent_violation_signal_rate: metrics
.summary
.pass_run_dsfb_persistent_violation_nuisance_rate,
dsfb_dsa_signal_rate: dsa.summary.pass_run_nuisance_proxy,
ewma_signal_rate: metrics.summary.pass_run_ewma_nuisance_rate,
cusum_signal_rate: metrics.summary.pass_run_cusum_nuisance_rate,
run_energy_signal_rate: metrics.summary.pass_run_run_energy_nuisance_rate,
pca_fdc_signal_rate: metrics.summary.pass_run_pca_fdc_nuisance_rate,
threshold_signal_rate: metrics.summary.pass_run_threshold_nuisance_rate,
},
lead_time_summary: metrics.lead_time_summary.clone(),
density_summary: metrics.density_summary.clone(),
boundary_episode_summary: metrics.boundary_episode_summary.clone(),
dsa_comparison_summary: Some(dsa.comparison_summary.clone()),
}
}
fn write_json_pretty<T: Serialize>(path: &Path, value: &T) -> Result<()> {
let json = serde_json::to_string_pretty(value)?;
fs::write(path, json)?;
Ok(())
}
fn write_feature_metrics_csv(path: &Path, metrics: &BenchmarkMetrics) -> Result<()> {
let mut writer = csv::Writer::from_path(path)?;
for feature in &metrics.feature_metrics {
writer.serialize(feature)?;
}
writer.flush()?;
Ok(())
}
fn write_per_failure_run_signals_csv(path: &Path, records: &[PerFailureRunSignal]) -> Result<()> {
let mut writer = csv::Writer::from_path(path)?;
for record in records {
writer.serialize(record)?;
}
writer.flush()?;
Ok(())
}
fn write_dsa_metrics_csv(
path: &Path,
prepared: &crate::preprocessing::PreparedDataset,
nominal: &crate::nominal::NominalModel,
dsa: &DsaEvaluation,
) -> Result<()> {
let mut writer = csv::Writer::from_path(path)?;
writer.write_record([
"feature_index",
"feature_name",
"run_index",
"timestamp",
"label",
"boundary_basis_hit",
"drift_outward_hit",
"slew_hit",
"motif_hit",
"boundary_density_W",
"drift_persistence_W",
"slew_density_W",
"ewma_occupancy_W",
"motif_recurrence_W",
"fragmentation_proxy_W",
"consistent",
"dsa_score",
"dsa_active",
"numeric_dsa_alert",
"dsa_alert",
"resolved_alert_class",
"policy_state",
"policy_suppressed_to_silent",
"rescue_transition",
"rescued_to_review",
])?;
for trace in &dsa.traces {
if !nominal.features[trace.feature_index].analyzable {
continue;
}
for run_index in 0..trace.dsa_score.len() {
writer.write_record([
trace.feature_index.to_string(),
trace.feature_name.clone(),
run_index.to_string(),
prepared.timestamps[run_index]
.format("%Y-%m-%d %H:%M:%S")
.to_string(),
prepared.labels[run_index].to_string(),
trace.boundary_basis_hit[run_index].to_string(),
trace.drift_outward_hit[run_index].to_string(),
trace.slew_hit[run_index].to_string(),
trace.motif_hit[run_index].to_string(),
trace.boundary_density_w[run_index].to_string(),
trace.drift_persistence_w[run_index].to_string(),
trace.slew_density_w[run_index].to_string(),
trace.ewma_occupancy_w[run_index].to_string(),
trace.motif_recurrence_w[run_index].to_string(),
trace.fragmentation_proxy_w[run_index].to_string(),
trace.consistent[run_index].to_string(),
trace.dsa_score[run_index].to_string(),
trace.dsa_active[run_index].to_string(),
trace.numeric_dsa_alert[run_index].to_string(),
trace.dsa_alert[run_index].to_string(),
format!("{:?}", trace.resolved_alert_class[run_index]),
trace.policy_state[run_index].as_lowercase().to_string(),
trace.policy_suppressed_to_silent[run_index].to_string(),
trace.rescue_transition[run_index].clone(),
trace.rescued_to_review[run_index].to_string(),
])?;
}
}
writer.flush()?;
Ok(())
}
fn write_dsa_run_signals_csv(
path: &Path,
prepared: &crate::preprocessing::PreparedDataset,
run_signals: &DsaRunSignals,
) -> Result<()> {
let mut writer = csv::Writer::from_path(path)?;
writer.write_record([
"run_index",
"timestamp",
"label",
"primary_run_signal",
"corroborating_feature_count_min",
"primary_run_alert",
"any_feature_dsa_alert",
"any_feature_raw_violation",
"feature_count_dsa_alert",
"watch_feature_count",
"review_feature_count",
"escalate_feature_count",
"strict_escalate_run_alert",
"numeric_primary_run_alert",
"numeric_feature_count_dsa_alert",
])?;
for run_index in 0..prepared.labels.len() {
writer.write_record([
run_index.to_string(),
prepared.timestamps[run_index]
.format("%Y-%m-%d %H:%M:%S")
.to_string(),
prepared.labels[run_index].to_string(),
run_signals.primary_run_signal.clone(),
run_signals.corroborating_feature_count_min.to_string(),
run_signals.primary_run_alert[run_index].to_string(),
run_signals.any_feature_dsa_alert[run_index].to_string(),
run_signals.any_feature_raw_violation[run_index].to_string(),
run_signals.feature_count_dsa_alert[run_index].to_string(),
run_signals.watch_feature_count[run_index].to_string(),
run_signals.review_feature_count[run_index].to_string(),
run_signals.escalate_feature_count[run_index].to_string(),
run_signals.strict_escalate_run_alert[run_index].to_string(),
run_signals.numeric_primary_run_alert[run_index].to_string(),
run_signals.numeric_feature_count_dsa_alert[run_index].to_string(),
])?;
}
writer.flush()?;
Ok(())
}
fn write_per_failure_run_dsa_signals_csv(
path: &Path,
records: &[PerFailureRunDsaSignal],
) -> Result<()> {
let mut writer = csv::Writer::from_path(path)?;
writer.write_record([
"failure_run_index",
"failure_timestamp",
"earliest_dsa_run",
"earliest_primary_source",
"earliest_dsa_feature_index",
"earliest_dsa_feature_name",
"dsa_lead_runs",
"threshold_lead_runs",
"ewma_lead_runs",
"cusum_lead_runs",
"run_energy_lead_runs",
"pca_fdc_lead_runs",
"dsa_minus_cusum_delta_runs",
"dsa_minus_run_energy_delta_runs",
"dsa_minus_pca_fdc_delta_runs",
"dsa_minus_threshold_delta_runs",
"dsa_minus_ewma_delta_runs",
"dsa_alerting_feature_count",
"max_dsa_score_in_lookback",
"max_dsa_score_feature_index",
"max_dsa_score_feature_name",
"max_dsa_score_run_index",
"max_dsa_score_boundary_density_w",
"max_dsa_score_drift_persistence_w",
"max_dsa_score_slew_density_w",
"max_dsa_score_ewma_occupancy_w",
"max_dsa_score_motif_recurrence_w",
"max_dsa_score_fragmentation_proxy_w",
"max_dsa_score_consistent",
"max_dsa_score_policy_state",
"max_dsa_score_resolved_alert_class",
"max_dsa_score_numeric_dsa_alert",
"max_dsa_score_dsa_alert",
"max_dsa_score_policy_suppressed",
"max_dsa_score_rescue_transition",
])?;
for record in records {
writer.write_record([
record.failure_run_index.to_string(),
record.failure_timestamp.clone(),
option_to_string(record.earliest_dsa_run),
record.earliest_primary_source.clone().unwrap_or_default(),
option_to_string(record.earliest_dsa_feature_index),
record.earliest_dsa_feature_name.clone().unwrap_or_default(),
option_to_string(record.dsa_lead_runs),
option_to_string(record.threshold_lead_runs),
option_to_string(record.ewma_lead_runs),
option_to_string(record.cusum_lead_runs),
option_to_string(record.run_energy_lead_runs),
option_to_string(record.pca_fdc_lead_runs),
option_to_string(record.dsa_minus_cusum_delta_runs),
option_to_string(record.dsa_minus_run_energy_delta_runs),
option_to_string(record.dsa_minus_pca_fdc_delta_runs),
option_to_string(record.dsa_minus_threshold_delta_runs),
option_to_string(record.dsa_minus_ewma_delta_runs),
record.dsa_alerting_feature_count.to_string(),
option_to_string(record.max_dsa_score_in_lookback),
option_to_string(record.max_dsa_score_feature_index),
record
.max_dsa_score_feature_name
.clone()
.unwrap_or_default(),
option_to_string(record.max_dsa_score_run_index),
option_to_string(record.max_dsa_score_boundary_density_w),
option_to_string(record.max_dsa_score_drift_persistence_w),
option_to_string(record.max_dsa_score_slew_density_w),
option_to_string(record.max_dsa_score_ewma_occupancy_w),
option_to_string(record.max_dsa_score_motif_recurrence_w),
option_to_string(record.max_dsa_score_fragmentation_proxy_w),
option_to_string(record.max_dsa_score_consistent),
record
.max_dsa_score_policy_state
.clone()
.unwrap_or_default(),
record
.max_dsa_score_resolved_alert_class
.clone()
.unwrap_or_default(),
option_to_string(record.max_dsa_score_numeric_dsa_alert),
option_to_string(record.max_dsa_score_dsa_alert),
option_to_string(record.max_dsa_score_policy_suppressed),
record
.max_dsa_score_rescue_transition
.clone()
.unwrap_or_default(),
])?;
}
writer.flush()?;
Ok(())
}
fn write_lead_time_metrics_csv(path: &Path, records: &[PerFailureRunSignal]) -> Result<()> {
let mut writer = csv::Writer::from_path(path)?;
writer.write_record([
"failure_run_index",
"failure_timestamp",
"earliest_dsfb_raw_boundary_run",
"earliest_dsfb_persistent_boundary_run",
"earliest_dsfb_raw_violation_run",
"earliest_dsfb_persistent_violation_run",
"earliest_threshold_run",
"earliest_ewma_run",
"earliest_cusum_run",
"earliest_run_energy_run",
"earliest_pca_fdc_run",
"dsfb_raw_boundary_lead_runs",
"dsfb_persistent_boundary_lead_runs",
"dsfb_raw_violation_lead_runs",
"dsfb_persistent_violation_lead_runs",
"threshold_lead_runs",
"ewma_lead_runs",
"cusum_lead_runs",
"run_energy_lead_runs",
"pca_fdc_lead_runs",
"dsfb_raw_boundary_minus_cusum_delta_runs",
"dsfb_raw_boundary_minus_run_energy_delta_runs",
"dsfb_raw_boundary_minus_pca_fdc_delta_runs",
"dsfb_raw_boundary_minus_threshold_delta_runs",
"dsfb_raw_boundary_minus_ewma_delta_runs",
"dsfb_persistent_boundary_minus_cusum_delta_runs",
"dsfb_persistent_boundary_minus_run_energy_delta_runs",
"dsfb_persistent_boundary_minus_pca_fdc_delta_runs",
"dsfb_persistent_boundary_minus_threshold_delta_runs",
"dsfb_persistent_boundary_minus_ewma_delta_runs",
"dsfb_raw_violation_minus_cusum_delta_runs",
"dsfb_raw_violation_minus_run_energy_delta_runs",
"dsfb_raw_violation_minus_pca_fdc_delta_runs",
"dsfb_raw_violation_minus_threshold_delta_runs",
"dsfb_raw_violation_minus_ewma_delta_runs",
"dsfb_persistent_violation_minus_cusum_delta_runs",
"dsfb_persistent_violation_minus_run_energy_delta_runs",
"dsfb_persistent_violation_minus_pca_fdc_delta_runs",
"dsfb_persistent_violation_minus_threshold_delta_runs",
"dsfb_persistent_violation_minus_ewma_delta_runs",
])?;
for record in records {
writer.write_record([
record.failure_run_index.to_string(),
record.failure_timestamp.clone(),
option_to_string(record.earliest_dsfb_raw_boundary_run),
option_to_string(record.earliest_dsfb_persistent_boundary_run),
option_to_string(record.earliest_dsfb_raw_violation_run),
option_to_string(record.earliest_dsfb_persistent_violation_run),
option_to_string(record.earliest_threshold_run),
option_to_string(record.earliest_ewma_run),
option_to_string(record.earliest_cusum_run),
option_to_string(record.earliest_run_energy_run),
option_to_string(record.earliest_pca_fdc_run),
option_to_string(record.dsfb_raw_boundary_lead_runs),
option_to_string(record.dsfb_persistent_boundary_lead_runs),
option_to_string(record.dsfb_raw_violation_lead_runs),
option_to_string(record.dsfb_persistent_violation_lead_runs),
option_to_string(record.threshold_lead_runs),
option_to_string(record.ewma_lead_runs),
option_to_string(record.cusum_lead_runs),
option_to_string(record.run_energy_lead_runs),
option_to_string(record.pca_fdc_lead_runs),
option_to_string(record.dsfb_raw_boundary_minus_cusum_delta_runs),
option_to_string(record.dsfb_raw_boundary_minus_run_energy_delta_runs),
option_to_string(record.dsfb_raw_boundary_minus_pca_fdc_delta_runs),
option_to_string(record.dsfb_raw_boundary_minus_threshold_delta_runs),
option_to_string(record.dsfb_raw_boundary_minus_ewma_delta_runs),
option_to_string(record.dsfb_persistent_boundary_minus_cusum_delta_runs),
option_to_string(record.dsfb_persistent_boundary_minus_run_energy_delta_runs),
option_to_string(record.dsfb_persistent_boundary_minus_pca_fdc_delta_runs),
option_to_string(record.dsfb_persistent_boundary_minus_threshold_delta_runs),
option_to_string(record.dsfb_persistent_boundary_minus_ewma_delta_runs),
option_to_string(record.dsfb_raw_violation_minus_cusum_delta_runs),
option_to_string(record.dsfb_raw_violation_minus_run_energy_delta_runs),
option_to_string(record.dsfb_raw_violation_minus_pca_fdc_delta_runs),
option_to_string(record.dsfb_raw_violation_minus_threshold_delta_runs),
option_to_string(record.dsfb_raw_violation_minus_ewma_delta_runs),
option_to_string(record.dsfb_persistent_violation_minus_cusum_delta_runs),
option_to_string(record.dsfb_persistent_violation_minus_run_energy_delta_runs),
option_to_string(record.dsfb_persistent_violation_minus_pca_fdc_delta_runs),
option_to_string(record.dsfb_persistent_violation_minus_threshold_delta_runs),
option_to_string(record.dsfb_persistent_violation_minus_ewma_delta_runs),
])?;
}
writer.flush()?;
Ok(())
}
fn write_density_metrics_csv(path: &Path, records: &[DensityMetricRecord]) -> Result<()> {
let mut writer = csv::Writer::from_path(path)?;
for record in records {
writer.serialize(record)?;
}
writer.flush()?;
Ok(())
}
fn write_dsfb_signs_csv(
path: &Path,
prepared: &crate::preprocessing::PreparedDataset,
residuals: &crate::residual::ResidualSet,
signs: &crate::signs::SignSet,
) -> Result<()> {
let mut writer = csv::Writer::from_path(path)?;
writer.write_record([
"feature_index",
"feature_name",
"run_index",
"timestamp",
"label",
"residual",
"drift",
"slew",
"residual_norm",
"is_imputed",
"drift_threshold",
"slew_threshold",
])?;
for (residual_trace, sign_trace) in residuals.traces.iter().zip(&signs.traces) {
for run_index in 0..residual_trace.residuals.len() {
writer.write_record([
residual_trace.feature_index.to_string(),
residual_trace.feature_name.clone(),
run_index.to_string(),
prepared.timestamps[run_index]
.format("%Y-%m-%d %H:%M:%S")
.to_string(),
prepared.labels[run_index].to_string(),
residual_trace.residuals[run_index].to_string(),
sign_trace.drift[run_index].to_string(),
sign_trace.slew[run_index].to_string(),
residual_trace.norms[run_index].to_string(),
residual_trace.is_imputed[run_index].to_string(),
sign_trace.drift_threshold.to_string(),
sign_trace.slew_threshold.to_string(),
])?;
}
}
writer.flush()?;
Ok(())
}
fn write_dsfb_motifs_csv(path: &Path, motifs: &crate::semiotics::MotifSet) -> Result<()> {
let mut writer = csv::Writer::from_path(path)?;
for row in &motifs.summary_rows {
writer.serialize(row)?;
}
writer.flush()?;
Ok(())
}
fn write_dsfb_motif_labels_per_time_csv(
path: &Path,
prepared: &crate::preprocessing::PreparedDataset,
motifs: &crate::semiotics::MotifSet,
) -> Result<()> {
let mut writer = csv::Writer::from_path(path)?;
writer.write_record([
"feature_index",
"feature_name",
"run_index",
"timestamp",
"label",
"motif_label",
])?;
for trace in &motifs.traces {
for (run_index, motif_label) in trace.labels.iter().enumerate() {
writer.write_record([
trace.feature_index.to_string(),
trace.feature_name.clone(),
run_index.to_string(),
prepared.timestamps[run_index]
.format("%Y-%m-%d %H:%M:%S")
.to_string(),
prepared.labels[run_index].to_string(),
motif_label.as_lowercase().to_string(),
])?;
}
}
writer.flush()?;
Ok(())
}
fn write_dsfb_grammar_states_csv(
path: &Path,
prepared: &crate::preprocessing::PreparedDataset,
grammar: &crate::grammar::GrammarSet,
) -> Result<()> {
let mut writer = csv::Writer::from_path(path)?;
writer.write_record([
"feature_index",
"feature_name",
"run_index",
"timestamp",
"label",
"raw_state",
"confirmed_state",
"persistent_boundary",
"persistent_violation",
"suppressed_by_imputation",
"raw_reason",
"confirmed_reason",
])?;
for trace in &grammar.traces {
for run_index in 0..trace.raw_states.len() {
writer.write_record([
trace.feature_index.to_string(),
trace.feature_name.clone(),
run_index.to_string(),
prepared.timestamps[run_index]
.format("%Y-%m-%d %H:%M:%S")
.to_string(),
prepared.labels[run_index].to_string(),
format!("{:?}", trace.raw_states[run_index]),
format!("{:?}", trace.states[run_index]),
trace.persistent_boundary[run_index].to_string(),
trace.persistent_violation[run_index].to_string(),
trace.suppressed_by_imputation[run_index].to_string(),
format!("{:?}", trace.raw_reasons[run_index]),
format!("{:?}", trace.reasons[run_index]),
])?;
}
}
writer.flush()?;
Ok(())
}
fn write_dsfb_semantic_matches_csv(
path: &Path,
rows: &[crate::semiotics::SemanticMatchRecord],
) -> Result<()> {
let mut writer = csv::Writer::from_path(path)?;
for row in rows {
writer.serialize(row)?;
}
writer.flush()?;
Ok(())
}
fn write_serialized_csv<T: Serialize>(path: &Path, rows: &[T]) -> Result<()> {
let mut writer = csv::Writer::from_path(path)?;
for row in rows {
writer.serialize(row)?;
}
writer.flush()?;
Ok(())
}
fn write_trace_csvs(
run_dir: &Path,
prepared: &crate::preprocessing::PreparedDataset,
residuals: &crate::residual::ResidualSet,
signs: &crate::signs::SignSet,
baselines: &crate::baselines::BaselineSet,
grammar: &crate::grammar::GrammarSet,
) -> Result<()> {
let mut residual_writer = csv::Writer::from_path(run_dir.join("residuals.csv"))?;
let mut drift_writer = csv::Writer::from_path(run_dir.join("drifts.csv"))?;
let mut slew_writer = csv::Writer::from_path(run_dir.join("slews.csv"))?;
let mut ewma_writer = csv::Writer::from_path(run_dir.join("ewma_baseline.csv"))?;
let mut cusum_writer = csv::Writer::from_path(run_dir.join("cusum_baseline.csv"))?;
let mut run_energy_writer = csv::Writer::from_path(run_dir.join("run_energy_baseline.csv"))?;
let mut pca_fdc_writer = csv::Writer::from_path(run_dir.join("pca_fdc_baseline.csv"))?;
let mut grammar_writer = csv::Writer::from_path(run_dir.join("grammar_states.csv"))?;
residual_writer.write_record([
"run_index",
"timestamp",
"label",
"feature",
"imputed_value",
"is_imputed",
"residual",
"residual_norm",
"threshold_alarm",
])?;
drift_writer.write_record(["run_index", "timestamp", "feature", "drift"])?;
slew_writer.write_record(["run_index", "timestamp", "feature", "slew"])?;
ewma_writer.write_record([
"run_index",
"timestamp",
"feature",
"ewma",
"healthy_mean",
"healthy_std",
"threshold",
"alarm",
])?;
cusum_writer.write_record([
"run_index",
"timestamp",
"feature",
"cusum",
"healthy_mean",
"healthy_std",
"kappa",
"alarm_threshold",
"alarm",
])?;
run_energy_writer.write_record([
"run_index",
"timestamp",
"label",
"run_energy",
"healthy_mean",
"healthy_std",
"threshold",
"analyzable_feature_count",
"alarm",
])?;
pca_fdc_writer.write_record([
"run_index",
"timestamp",
"label",
"pca_t2",
"pca_t2_healthy_mean",
"pca_t2_healthy_std",
"pca_t2_threshold",
"pca_spe",
"pca_spe_healthy_mean",
"pca_spe_healthy_std",
"pca_spe_threshold",
"retained_components",
"explained_variance_fraction",
"target_variance_explained",
"analyzable_feature_count",
"alarm",
])?;
grammar_writer.write_record([
"run_index",
"timestamp",
"feature",
"raw_state",
"confirmed_state",
"persistent_boundary",
"persistent_violation",
"suppressed_by_imputation",
"raw_reason",
"confirmed_reason",
])?;
for feature_index in 0..residuals.traces.len() {
let residual_trace = &residuals.traces[feature_index];
let sign_trace = &signs.traces[feature_index];
let ewma_trace = &baselines.ewma[feature_index];
let cusum_trace = &baselines.cusum[feature_index];
let grammar_trace = &grammar.traces[feature_index];
for run_index in 0..prepared.timestamps.len() {
let timestamp = prepared.timestamps[run_index]
.format("%Y-%m-%d %H:%M:%S")
.to_string();
residual_writer.write_record([
run_index.to_string(),
timestamp.clone(),
prepared.labels[run_index].to_string(),
residual_trace.feature_name.clone(),
residual_trace.imputed_values[run_index].to_string(),
residual_trace.is_imputed[run_index].to_string(),
residual_trace.residuals[run_index].to_string(),
residual_trace.norms[run_index].to_string(),
residual_trace.threshold_alarm[run_index].to_string(),
])?;
drift_writer.write_record([
run_index.to_string(),
timestamp.clone(),
residual_trace.feature_name.clone(),
sign_trace.drift[run_index].to_string(),
])?;
slew_writer.write_record([
run_index.to_string(),
timestamp.clone(),
residual_trace.feature_name.clone(),
sign_trace.slew[run_index].to_string(),
])?;
ewma_writer.write_record([
run_index.to_string(),
timestamp.clone(),
residual_trace.feature_name.clone(),
ewma_trace.ewma[run_index].to_string(),
ewma_trace.healthy_mean.to_string(),
ewma_trace.healthy_std.to_string(),
ewma_trace.threshold.to_string(),
ewma_trace.alarm[run_index].to_string(),
])?;
cusum_writer.write_record([
run_index.to_string(),
timestamp.clone(),
residual_trace.feature_name.clone(),
cusum_trace.cusum[run_index].to_string(),
cusum_trace.healthy_mean.to_string(),
cusum_trace.healthy_std.to_string(),
cusum_trace.kappa.to_string(),
cusum_trace.alarm_threshold.to_string(),
cusum_trace.alarm[run_index].to_string(),
])?;
grammar_writer.write_record([
run_index.to_string(),
timestamp,
residual_trace.feature_name.clone(),
format!("{:?}", grammar_trace.raw_states[run_index]),
format!("{:?}", grammar_trace.states[run_index]),
grammar_trace.persistent_boundary[run_index].to_string(),
grammar_trace.persistent_violation[run_index].to_string(),
grammar_trace.suppressed_by_imputation[run_index].to_string(),
format!("{:?}", grammar_trace.raw_reasons[run_index]),
format!("{:?}", grammar_trace.reasons[run_index]),
])?;
}
}
for run_index in 0..prepared.timestamps.len() {
run_energy_writer.write_record([
run_index.to_string(),
prepared.timestamps[run_index]
.format("%Y-%m-%d %H:%M:%S")
.to_string(),
prepared.labels[run_index].to_string(),
baselines.run_energy.energy[run_index].to_string(),
baselines.run_energy.healthy_mean.to_string(),
baselines.run_energy.healthy_std.to_string(),
baselines.run_energy.threshold.to_string(),
baselines.run_energy.analyzable_feature_count.to_string(),
baselines.run_energy.alarm[run_index].to_string(),
])?;
pca_fdc_writer.write_record([
run_index.to_string(),
prepared.timestamps[run_index]
.format("%Y-%m-%d %H:%M:%S")
.to_string(),
prepared.labels[run_index].to_string(),
baselines.pca_fdc.t2[run_index].to_string(),
baselines.pca_fdc.t2_healthy_mean.to_string(),
baselines.pca_fdc.t2_healthy_std.to_string(),
baselines.pca_fdc.t2_threshold.to_string(),
baselines.pca_fdc.spe[run_index].to_string(),
baselines.pca_fdc.spe_healthy_mean.to_string(),
baselines.pca_fdc.spe_healthy_std.to_string(),
baselines.pca_fdc.spe_threshold.to_string(),
baselines.pca_fdc.retained_components.to_string(),
baselines.pca_fdc.explained_variance_fraction.to_string(),
baselines.pca_fdc.target_variance_explained.to_string(),
baselines.pca_fdc.analyzable_feature_count.to_string(),
baselines.pca_fdc.alarm[run_index].to_string(),
])?;
}
residual_writer.flush()?;
drift_writer.flush()?;
slew_writer.flush()?;
ewma_writer.flush()?;
cusum_writer.flush()?;
run_energy_writer.flush()?;
pca_fdc_writer.flush()?;
grammar_writer.flush()?;
Ok(())
}
fn option_to_string<T: ToString>(value: Option<T>) -> String {
value.map(|value| value.to_string()).unwrap_or_default()
}
fn zip_directory(run_dir: &Path, zip_path: &Path) -> Result<()> {
let file = File::create(zip_path)?;
let mut zip = zip::ZipWriter::new(file);
let options = SimpleFileOptions::default()
.compression_method(zip::CompressionMethod::Deflated)
.unix_permissions(0o644);
add_directory_contents(&mut zip, run_dir, run_dir, zip_path, options)?;
zip.finish()?;
Ok(())
}
fn add_directory_contents(
zip: &mut zip::ZipWriter<File>,
root: &Path,
current: &Path,
zip_path: &Path,
options: SimpleFileOptions,
) -> Result<()> {
for entry in fs::read_dir(current)? {
let entry = entry?;
let path = entry.path();
if path == zip_path {
continue;
}
if path.is_dir() {
add_directory_contents(zip, root, &path, zip_path, options)?;
} else {
let relative = path
.strip_prefix(root)
.map_err(|err| DsfbSemiconductorError::DatasetFormat(err.to_string()))?;
zip.start_file(relative.to_string_lossy().replace('\\', "/"), options)?;
let bytes = fs::read(&path)?;
zip.write_all(&bytes)?;
}
}
Ok(())
}