1use crate::baselines::compute_baselines;
2use chrono::Utc;
3use crate::cohort::{
4 build_seed_feature_check, compute_rating_delta_forecast, compute_rating_failure_analysis,
5 run_recall_optimization, write_cohort_results_csv,
6 write_failure_analysis_md as write_cohort_failure_analysis_md,
7 write_feature_policy_summary_csv, write_feature_ranking_comparison_csv,
8 write_feature_ranking_csv, write_heuristic_policy_failure_analysis_md,
9 write_missed_failure_diagnostics_csv, write_motif_policy_contributions_csv,
10 write_operator_burden_contributions_csv, write_operator_delta_attainment_matrix_csv,
11 write_policy_contribution_analysis_csv, write_precursor_quality_csv,
12 write_recall_critical_features_csv, write_recall_recovery_efficiency_csv,
13 write_recall_rescue_results_csv, write_single_change_iteration_log_csv,
14};
15use crate::config::{PipelineConfig, RunConfiguration};
16use crate::dataset::phm2018::{support_status as phm_support_status, Phm2018SupportStatus};
17use crate::dataset::secom::{self, SecomArchiveLayout};
18use crate::error::{DsfbSemiconductorError, Result};
19use crate::failure_driven::build_failure_driven_artifacts;
20use crate::grammar::evaluate_grammar;
21use crate::heuristics::build_heuristics_bank;
22use crate::metrics::{
23 compute_metrics, BenchmarkMetrics, BoundaryEpisodeSummary, DensityMetricRecord, DensitySummary,
24 LeadTimeSummary, PerFailureRunSignal,
25};
26use crate::nominal::build_nominal_model;
27use crate::non_intrusive::materialize_non_intrusive_artifacts;
28use crate::output_paths::{create_timestamped_run_dir, default_output_root};
29use crate::plots::{generate_figures, FigureManifest};
30use crate::precursor::{
31 DsaEvaluation, DsaRunSignals, DsaVsBaselinesSummary, PerFailureRunDsaSignal,
32};
33use crate::preprocessing::prepare_secom;
34use crate::report::{write_reports, ReportArtifacts};
35use crate::residual::compute_residuals;
36use crate::secom_addendum::{
37 build_secom_addendum_artifacts, draw_recurrent_boundary_tradeoff_plot,
38};
39use crate::semiotics::{build_scaffold_semiotics, build_semantic_layer, classify_motifs};
40use crate::signs::compute_signs;
41use crate::traceability::{build_traceability_entries, write_traceability_json, DsfbRunManifest};
42use serde::Serialize;
43use std::fs::{self, File};
44use std::io::Write;
45use std::path::{Path, PathBuf};
46use zip::write::SimpleFileOptions;
47
48#[derive(Debug, Clone)]
51pub struct PaperLockMetrics {
52 pub episode_count: usize,
54 pub precision: f64,
56 pub detected_failures: usize,
58 pub total_failures: usize,
60}
61
62#[derive(Debug, Clone)]
63pub struct SecomRunArtifacts {
64 pub run_dir: PathBuf,
65 pub report: ReportArtifacts,
66 pub figures: FigureManifest,
67 pub metrics_path: PathBuf,
68 pub manifest_path: PathBuf,
69 pub zip_path: PathBuf,
70 pub phm2018_status: Phm2018SupportStatus,
71 pub paper_lock_metrics: PaperLockMetrics,
72}
73
74#[derive(Debug, Clone, Serialize)]
75struct ArtifactManifest {
76 dataset: String,
77 run_dir: String,
78 metrics_summary_path: String,
79 baseline_comparison_summary_path: String,
80 dsa_vs_baselines_summary_path: String,
81 dsa_parameter_manifest_path: String,
82 dsa_grid_results_path: String,
83 dsa_grid_summary_path: String,
84 dsa_feature_ranking_path: String,
85 dsa_feature_ranking_recall_aware_path: String,
86 dsa_feature_ranking_dsfb_aware_path: String,
87 dsa_feature_ranking_comparison_path: String,
88 dsa_seed_feature_check_path: String,
89 dsa_feature_cohorts_path: String,
90 dsa_feature_policy_overrides_path: String,
91 dsa_feature_policy_summary_path: String,
92 dsa_recall_rescue_results_path: String,
93 dsa_recall_critical_features_path: String,
94 dsa_pareto_frontier_path: String,
95 dsa_stage_a_candidates_path: String,
96 dsa_stage_b_candidates_path: String,
97 dsa_missed_failure_diagnostics_path: String,
98 dsa_delta_target_assessment_path: String,
99 dsa_cohort_results_path: String,
100 dsa_cohort_results_recall_aware_path: String,
101 dsa_cohort_results_dsfb_aware_path: String,
102 dsa_cohort_summary_path: String,
103 dsa_cohort_summary_recall_aware_path: String,
104 dsa_cohort_summary_dsfb_aware_path: String,
105 dsa_cohort_precursor_quality_path: String,
106 dsa_cohort_failure_analysis_path: Option<String>,
107 dsa_heuristic_policy_failure_analysis_path: Option<String>,
108 dsa_motif_policy_contributions_path: String,
109 dsa_policy_contribution_analysis_path: String,
110 dsa_rating_delta_forecast_path: String,
111 dsa_rating_delta_failure_analysis_path: Option<String>,
112 lead_time_metrics_path: String,
113 density_metrics_path: String,
114 cusum_baseline_path: String,
115 run_energy_baseline_path: String,
116 pca_fdc_baseline_path: String,
117 per_failure_run_signals_path: String,
118 dsa_metrics_path: String,
119 dsa_run_signals_path: String,
120 per_failure_run_dsa_signals_path: String,
121 dsfb_signs_path: String,
122 dsfb_feature_signs_path: String,
123 dsfb_motifs_path: String,
124 dsfb_motif_labels_per_time_path: String,
125 dsfb_feature_motif_timeline_path: String,
126 dsfb_grammar_states_path: String,
127 dsfb_feature_grammar_states_path: String,
128 dsfb_envelope_interaction_summary_path: String,
129 dsfb_heuristics_bank_expanded_path: String,
130 dsfb_semantic_matches_path: String,
131 dsfb_semantic_ranked_candidates_path: String,
132 dsfb_feature_policy_decisions_path: String,
133 dsfb_traceability_path: String,
134 dsfb_group_definitions_path: String,
135 dsfb_group_signs_path: String,
136 dsfb_group_grammar_states_path: String,
137 dsfb_group_semantic_matches_path: String,
138 dsfb_structural_delta_metrics_path: String,
139 recurrent_boundary_stats_path: String,
140 recurrent_boundary_tradeoff_curve_path: String,
141 recurrent_boundary_tradeoff_plot_path: String,
142 dsfb_metric_regrounding_path: String,
143 target_d_regression_analysis_path: String,
144 missed_failure_root_cause_path: String,
145 lead_time_comparison_path: String,
146 lead_time_explanation_path: String,
147 episode_precision_metrics_path: String,
148 dsfb_episode_summary_path: String,
149 dsfb_episode_precision_path: String,
150 dsfb_recall_metrics_path: String,
151 paper_abstract_artifact_path: String,
152 dsa_operator_baselines_path: String,
153 dsa_operator_delta_targets_path: String,
154 dsa_operator_delta_attainment_matrix_path: String,
155 dsa_policy_operator_burden_contributions_path: String,
156 dsa_recall_recovery_efficiency_path: String,
157 dsfb_single_change_iteration_log_path: String,
158 optimization_log_path: String,
159 failures_index_path: String,
160 missed_failure_priority_path: String,
161 failure_case_paths: Vec<String>,
162 feature_motif_grounding_path: String,
163 feature_to_motif_path: String,
164 negative_control_report_path: String,
165 dsfb_heuristics_bank_minimal_path: String,
166 dsfb_heuristic_provenance_path: String,
167 policy_decisions_path: String,
168 policy_burden_summary_path: String,
169 dsfb_feature_role_validation_path: String,
170 dsfb_group_validation_path: String,
171 dsfb_vs_ewma_case_paths: Vec<String>,
172 dsa_stage1_candidates_path: String,
173 dsa_stage2_candidates_path: String,
174 dsa_feature_ranking_burden_aware_path: String,
175 dsa_cohort_results_burden_aware_path: String,
176 dsa_cohort_summary_burden_aware_path: String,
177 secom_archive_layout_path: String,
178 drsc_trace_path: Option<String>,
179 drsc_figure_path: Option<String>,
180 drsc_dsa_combined_trace_path: Option<String>,
181 drsc_dsa_combined_figure_path: Option<String>,
182 dsa_focus_trace_path: Option<String>,
183 dsa_focus_figure_path: Option<String>,
184 non_intrusive_interface_spec_path: String,
185 non_intrusive_architecture_png_path: String,
186 non_intrusive_architecture_svg_path: String,
187 report_markdown_path: String,
188 report_tex_path: String,
189 report_pdf_path: Option<String>,
190 report_pdf_alias_path: Option<String>,
191 zip_path: String,
192 results_zip_path: String,
193}
194
195#[derive(Debug, Clone, Serialize)]
196struct BaselineComparisonSummary {
197 dataset: String,
198 secom_archive_layout_note: String,
199 feature_count_used_by_crate: usize,
200 failure_runs: usize,
201 analyzable_feature_count: usize,
202 grammar_imputation_suppression_points: usize,
203 lookback_runs: usize,
204 failure_run_recall: FailureRunRecallSummary,
205 pass_run_nuisance_proxy: PassRunNuisanceSummary,
206 lead_time_summary: LeadTimeSummary,
207 density_summary: DensitySummary,
208 boundary_episode_summary: BoundaryEpisodeSummary,
209 dsa_comparison_summary: Option<DsaVsBaselinesSummary>,
210}
211
212#[derive(Debug, Clone, Serialize)]
213struct FailureRunRecallSummary {
214 dsfb_raw_signal: usize,
215 dsfb_persistent_signal: usize,
216 dsfb_raw_boundary_signal: usize,
217 dsfb_persistent_boundary_signal: usize,
218 dsfb_raw_violation_signal: usize,
219 dsfb_persistent_violation_signal: usize,
220 dsfb_dsa_signal: usize,
221 ewma_signal: usize,
222 cusum_signal: usize,
223 run_energy_signal: usize,
224 pca_fdc_signal: usize,
225 threshold_signal: usize,
226}
227
228#[derive(Debug, Clone, Serialize)]
229struct PassRunNuisanceSummary {
230 dsfb_raw_boundary_signal_runs: usize,
231 dsfb_persistent_boundary_signal_runs: usize,
232 dsfb_raw_violation_signal_runs: usize,
233 dsfb_persistent_violation_signal_runs: usize,
234 dsfb_dsa_signal_runs: usize,
235 ewma_signal_runs: usize,
236 cusum_signal_runs: usize,
237 run_energy_signal_runs: usize,
238 pca_fdc_signal_runs: usize,
239 threshold_signal_runs: usize,
240 dsfb_raw_boundary_signal_rate: f64,
241 dsfb_persistent_boundary_signal_rate: f64,
242 dsfb_raw_violation_signal_rate: f64,
243 dsfb_persistent_violation_signal_rate: f64,
244 dsfb_dsa_signal_rate: f64,
245 ewma_signal_rate: f64,
246 cusum_signal_rate: f64,
247 run_energy_signal_rate: f64,
248 pca_fdc_signal_rate: f64,
249 threshold_signal_rate: f64,
250}
251
252#[derive(Debug, Clone, Serialize)]
253struct DsfbEpisodeSummaryRow {
254 raw_boundary_episodes: usize,
255 dsfb_episodes: usize,
256 episode_collapse_fraction: f64,
257}
258
259#[derive(Debug, Clone, Serialize)]
260struct DsfbEpisodePrecisionRow {
261 raw_boundary_precision: f64,
262 dsfb_precision: f64,
263 precision_gain_factor: f64,
264 raw_boundary_episodes: usize,
265 dsfb_episodes: usize,
266 dsfb_pre_failure_episodes: usize,
267}
268
269#[derive(Debug, Clone, Serialize)]
270struct DsfbRecallMetricsRow {
271 detected_failures: usize,
272 total_failures: usize,
273 recall: f64,
274}
275
276pub fn run_secom_benchmark(
277 data_root: &Path,
278 output_root: Option<&Path>,
279 config: PipelineConfig,
280 fetch_if_missing: bool,
281) -> Result<SecomRunArtifacts> {
282 config
283 .validate()
284 .map_err(DsfbSemiconductorError::DatasetFormat)?;
285
286 let paths = if fetch_if_missing {
287 secom::fetch_if_missing(data_root)?
288 } else {
289 secom::ensure_present(data_root)?
290 };
291 let secom_archive_layout = secom::inspect_archive_layout(&paths)?;
292 let dataset = secom::load_from_root(data_root)?;
293 let prepared = prepare_secom(&dataset, &config)?;
294 let nominal = build_nominal_model(&prepared, &config);
295 let residuals = compute_residuals(&prepared, &nominal);
296 let signs = compute_signs(&prepared, &nominal, &residuals, &config);
297 let baselines = compute_baselines(&prepared, &nominal, &residuals, &config);
298 let grammar = evaluate_grammar(&residuals, &signs, &nominal, &config);
299 let mut metrics = compute_metrics(
300 &prepared, &nominal, &residuals, &signs, &baselines, &grammar, &config,
301 );
302 let heuristics = build_heuristics_bank(&metrics, "SECOM");
303 let motifs = classify_motifs(
304 &prepared,
305 &nominal,
306 &residuals,
307 &signs,
308 &grammar,
309 config.pre_failure_lookback_runs,
310 );
311 let mut semantic_layer = build_semantic_layer(
312 &prepared,
313 &residuals,
314 &signs,
315 &grammar,
316 &motifs,
317 &nominal,
318 config.pre_failure_lookback_runs,
319 );
320 let scaffold_semiotics = build_scaffold_semiotics(
321 &prepared,
322 &nominal,
323 &residuals,
324 &grammar,
325 &motifs,
326 &semantic_layer,
327 );
328 let optimization = run_recall_optimization(
329 &prepared,
330 &nominal,
331 &residuals,
332 &signs,
333 &baselines,
334 &grammar,
335 &metrics,
336 &semantic_layer,
337 &scaffold_semiotics,
338 config.pre_failure_lookback_runs,
339 )?;
340 let selected_strategy = optimization
341 .optimized_execution
342 .summary
343 .selected_configuration
344 .as_ref()
345 .map(|row| row.ranking_strategy.as_str())
346 .unwrap_or("compression_biased");
347 let feature_ranking = match selected_strategy {
348 "recall_aware" => optimization.recall_aware_feature_ranking.clone(),
349 "burden_aware" => optimization.burden_aware_feature_ranking.clone(),
350 "dsfb_aware" => optimization.dsfb_aware_feature_ranking.clone(),
351 _ => optimization.baseline_feature_ranking.clone(),
352 };
353 let feature_cohorts = match selected_strategy {
354 "recall_aware" => optimization.recall_aware_feature_cohorts.clone(),
355 "burden_aware" => optimization.burden_aware_feature_cohorts.clone(),
356 "dsfb_aware" => optimization.dsfb_aware_feature_cohorts.clone(),
357 _ => optimization.baseline_feature_cohorts.clone(),
358 };
359 let seed_feature_check = build_seed_feature_check(&feature_cohorts);
360 let dsa = optimization.optimized_execution.selected_evaluation.clone();
361 let dsa_grid_summary = optimization.optimized_execution.grid_summary.clone();
362 let cohort_summary = optimization.optimized_execution.summary.clone();
363 metrics.dsa_summary = Some(dsa.summary.clone());
364 semantic_layer.structural_delta_metrics.episode_precision =
365 dsa.episode_summary.precursor_quality;
366 semantic_layer.structural_delta_metrics.compression_ratio =
367 dsa.episode_summary.compression_ratio;
368 let rating_delta_forecast =
369 compute_rating_delta_forecast(&dsa, &metrics, Some(&cohort_summary));
370 let rating_delta_failure_analysis =
371 compute_rating_failure_analysis(&dsa, &metrics, Some(&cohort_summary));
372 let failure_driven = build_failure_driven_artifacts(
373 &prepared,
374 &residuals,
375 &signs,
376 &baselines,
377 &grammar,
378 &motifs,
379 &semantic_layer,
380 &scaffold_semiotics,
381 &metrics,
382 &optimization.baseline_execution.selected_evaluation,
383 &dsa,
384 &optimization.missed_failure_diagnostics,
385 &optimization.policy_operator_burden_contributions,
386 config.pre_failure_lookback_runs,
387 );
388 let secom_addendum = build_secom_addendum_artifacts(
389 &prepared,
390 &residuals,
391 &baselines,
392 &grammar,
393 &motifs,
394 &semantic_layer,
395 &metrics,
396 &optimization,
397 &failure_driven,
398 &optimization.baseline_execution.selected_evaluation,
399 &dsa,
400 config.pre_failure_lookback_runs,
401 );
402
403 let paper_lock_metrics = PaperLockMetrics {
404 episode_count: secom_addendum.episode_precision_metrics.dsfb_episode_count,
405 precision: secom_addendum.episode_precision_metrics.dsfb_precision,
406 detected_failures: optimization
407 .operator_delta_targets
408 .selected_configuration
409 .failure_recall,
410 total_failures: optimization
411 .operator_delta_targets
412 .selected_configuration
413 .failure_runs,
414 };
415
416 let output_root = output_root
417 .map(Path::to_path_buf)
418 .unwrap_or_else(default_output_root);
419 fs::create_dir_all(&output_root)?;
420 let run_dir = create_timestamped_run_dir(&output_root, "secom")?;
421 let non_intrusive_artifacts = materialize_non_intrusive_artifacts(&run_dir)?;
422 write_readme_first(&run_dir, &config)?;
423 write_operator_summary(&run_dir, &config)?;
424
425 write_json_pretty(&run_dir.join("dataset_summary.json"), &prepared.summary)?;
426 write_json_pretty(&run_dir.join("parameter_manifest.json"), &config)?;
427 write_json_pretty(
428 &run_dir.join("run_configuration.json"),
429 &RunConfiguration {
430 dataset: "SECOM".into(),
431 config: config.clone(),
432 data_root: data_root.display().to_string(),
433 output_root: output_root.display().to_string(),
434 secom_fetch_if_missing: fetch_if_missing,
435 },
436 )?;
437 write_json_pretty(&run_dir.join("benchmark_metrics.json"), &metrics)?;
438 write_json_pretty(
439 &run_dir.join("secom_archive_layout.json"),
440 &secom_archive_layout,
441 )?;
442 write_json_pretty(
443 &run_dir.join("phm2018_support_status.json"),
444 &phm_support_status(data_root),
445 )?;
446 write_json_pretty(&run_dir.join("heuristics_bank.json"), &heuristics)?;
447 write_json_pretty(
448 &run_dir.join("baseline_comparison_summary.json"),
449 &build_baseline_comparison_summary(&metrics, &dsa, &secom_archive_layout, &config),
450 )?;
451 write_json_pretty(
452 &run_dir.join("dsa_vs_baselines.json"),
453 &dsa.comparison_summary,
454 )?;
455 write_json_pretty(
456 &run_dir.join("dsa_parameter_manifest.json"),
457 &dsa.parameter_manifest,
458 )?;
459 write_json_pretty(&run_dir.join("dsa_grid_summary.json"), &dsa_grid_summary)?;
460 write_feature_ranking_csv(&run_dir.join("dsa_feature_ranking.csv"), &feature_ranking)?;
461 write_feature_ranking_csv(
462 &run_dir.join("dsa_feature_ranking_recall_aware.csv"),
463 &optimization.recall_aware_feature_ranking,
464 )?;
465 write_feature_ranking_csv(
466 &run_dir.join("dsa_feature_ranking_dsfb_aware.csv"),
467 &optimization.dsfb_aware_feature_ranking,
468 )?;
469 write_feature_ranking_csv(
470 &run_dir.join("dsa_feature_ranking_burden_aware.csv"),
471 &optimization.burden_aware_feature_ranking,
472 )?;
473 write_feature_ranking_comparison_csv(
474 &run_dir.join("dsa_feature_ranking_comparison.csv"),
475 &optimization.ranking_comparison,
476 )?;
477 write_json_pretty(
478 &run_dir.join("dsa_seed_feature_check.json"),
479 &seed_feature_check,
480 )?;
481 write_json_pretty(&run_dir.join("dsa_feature_cohorts.json"), &feature_cohorts)?;
482 write_json_pretty(
483 &run_dir.join("dsa_feature_policy_overrides.json"),
484 &optimization.feature_policy_overrides,
485 )?;
486 write_feature_policy_summary_csv(
487 &run_dir.join("dsa_feature_policy_summary.csv"),
488 &optimization.feature_policy_summary,
489 )?;
490 write_recall_rescue_results_csv(
491 &run_dir.join("dsa_recall_rescue_results.csv"),
492 &optimization.recall_rescue_results,
493 )?;
494 write_recall_critical_features_csv(
495 &run_dir.join("dsa_recall_critical_features.csv"),
496 &optimization.recall_critical_features,
497 )?;
498 write_cohort_results_csv(
499 &run_dir.join("dsa_pareto_frontier.csv"),
500 &optimization.pareto_frontier,
501 )?;
502 write_cohort_results_csv(
503 &run_dir.join("dsa_stage_a_candidates.csv"),
504 &optimization.stage_a_candidates,
505 )?;
506 write_cohort_results_csv(
507 &run_dir.join("dsa_stage_b_candidates.csv"),
508 &optimization.stage_b_candidates,
509 )?;
510 write_missed_failure_diagnostics_csv(
511 &run_dir.join("dsa_missed_failure_diagnostics.csv"),
512 &optimization.missed_failure_diagnostics,
513 )?;
514 write_cohort_results_csv(
515 &run_dir.join("dsa_cohort_results.csv"),
516 &cohort_summary.cohort_results,
517 )?;
518 write_cohort_results_csv(
519 &run_dir.join("dsa_grid_results.csv"),
520 &cohort_summary.cohort_results,
521 )?;
522 write_cohort_results_csv(
523 &run_dir.join("dsa_cohort_results_recall_aware.csv"),
524 &optimization.recall_aware_execution.summary.cohort_results,
525 )?;
526 write_cohort_results_csv(
527 &run_dir.join("dsa_cohort_results_dsfb_aware.csv"),
528 &optimization.dsfb_aware_execution.summary.cohort_results,
529 )?;
530 write_cohort_results_csv(
531 &run_dir.join("dsa_cohort_results_burden_aware.csv"),
532 &optimization.burden_aware_execution.summary.cohort_results,
533 )?;
534 write_json_pretty(&run_dir.join("dsa_cohort_summary.json"), &cohort_summary)?;
535 write_json_pretty(
536 &run_dir.join("dsa_cohort_summary_recall_aware.json"),
537 &optimization.recall_aware_execution.summary,
538 )?;
539 write_json_pretty(
540 &run_dir.join("dsa_cohort_summary_dsfb_aware.json"),
541 &optimization.dsfb_aware_execution.summary,
542 )?;
543 write_json_pretty(
544 &run_dir.join("dsa_cohort_summary_burden_aware.json"),
545 &optimization.burden_aware_execution.summary,
546 )?;
547 write_precursor_quality_csv(
548 &run_dir.join("dsa_cohort_precursor_quality.csv"),
549 &cohort_summary.cohort_results,
550 )?;
551 write_motif_policy_contributions_csv(
552 &run_dir.join("dsa_motif_policy_contributions.csv"),
553 &optimization.optimized_execution.motif_policy_contributions,
554 )?;
555 write_policy_contribution_analysis_csv(
556 &run_dir.join("dsa_policy_contribution_analysis.csv"),
557 &optimization.policy_contribution_analysis,
558 )?;
559 write_json_pretty(
560 &run_dir.join("dsa_rating_delta_forecast.json"),
561 &rating_delta_forecast,
562 )?;
563 write_json_pretty(
564 &run_dir.join("dsa_delta_target_assessment.json"),
565 &optimization.delta_target_assessment,
566 )?;
567 write_json_pretty(
568 &run_dir.join("dsa_operator_baselines.json"),
569 &optimization.operator_baselines,
570 )?;
571 write_json_pretty(
572 &run_dir.join("dsa_operator_delta_targets.json"),
573 &optimization.operator_delta_targets,
574 )?;
575 write_operator_delta_attainment_matrix_csv(
576 &run_dir.join("dsa_operator_delta_attainment_matrix.csv"),
577 &optimization.operator_delta_attainment_matrix,
578 )?;
579 write_operator_burden_contributions_csv(
580 &run_dir.join("dsa_policy_operator_burden_contributions.csv"),
581 &optimization.policy_operator_burden_contributions,
582 )?;
583 write_recall_recovery_efficiency_csv(
584 &run_dir.join("dsa_recall_recovery_efficiency.csv"),
585 &optimization.recall_recovery_efficiency,
586 )?;
587 write_single_change_iteration_log_csv(
588 &run_dir.join("dsfb_single_change_iteration_log.csv"),
589 &optimization.single_change_iteration_log,
590 )?;
591 write_json_pretty(
592 &run_dir.join("optimization_log.json"),
593 &optimization.single_change_iteration_log,
594 )?;
595 write_json_pretty(
596 &run_dir.join("failures_index.json"),
597 &failure_driven.failures_index,
598 )?;
599 write_serialized_csv(
600 &run_dir.join("missed_failure_priority.csv"),
601 &failure_driven.missed_failure_priority,
602 )?;
603 for failure_case in &failure_driven.failure_cases {
604 write_json_pretty(
605 &run_dir.join(format!("failure_case_{}.json", failure_case.failure_id)),
606 failure_case,
607 )?;
608 }
609 write_json_pretty(
610 &run_dir.join("feature_motif_grounding.json"),
611 &failure_driven.feature_motif_grounding,
612 )?;
613 write_json_pretty(
614 &run_dir.join("feature_to_motif.json"),
615 &failure_driven.feature_to_motif,
616 )?;
617 write_json_pretty(
618 &run_dir.join("negative_control_report.json"),
619 &failure_driven.negative_control_report,
620 )?;
621 write_json_pretty(
622 &run_dir.join("dsfb_heuristics_bank_minimal.json"),
623 &failure_driven.minimal_heuristics_bank,
624 )?;
625 write_serialized_csv(
626 &run_dir.join("dsfb_heuristic_provenance.csv"),
627 &failure_driven.heuristic_provenance,
628 )?;
629 write_serialized_csv(
630 &run_dir.join("policy_burden_summary.csv"),
631 &failure_driven.policy_burden_summary,
632 )?;
633 write_serialized_csv(
634 &run_dir.join("dsfb_feature_role_validation.csv"),
635 &failure_driven.feature_role_validation,
636 )?;
637 write_serialized_csv(
638 &run_dir.join("dsfb_group_validation.csv"),
639 &failure_driven.group_validation,
640 )?;
641 for case in &failure_driven.dsfb_vs_ewma_cases {
642 write_json_pretty(
643 &run_dir.join(format!("dsfb_vs_ewma_case_{}.json", case.failure_id)),
644 case,
645 )?;
646 }
647 write_cohort_results_csv(
648 &run_dir.join("dsa_stage1_candidates.csv"),
649 &optimization.stage1_candidates,
650 )?;
651 write_cohort_results_csv(
652 &run_dir.join("dsa_stage2_candidates.csv"),
653 &optimization.stage2_candidates,
654 )?;
655 if let Some(analysis) = &cohort_summary.failure_analysis {
656 write_cohort_failure_analysis_md(
657 &run_dir.join("dsa_cohort_failure_analysis.md"),
658 analysis,
659 )?;
660 write_heuristic_policy_failure_analysis_md(
661 &run_dir.join("dsa_heuristic_policy_failure_analysis.md"),
662 analysis,
663 )?;
664 }
665 if let Some(analysis) = &rating_delta_failure_analysis {
666 crate::cohort::write_rating_failure_analysis_md(
667 &run_dir.join("dsa_rating_delta_failure_analysis.md"),
668 analysis,
669 )?;
670 }
671
672 write_feature_metrics_csv(&run_dir.join("feature_metrics.csv"), &metrics)?;
673 write_per_failure_run_signals_csv(
674 &run_dir.join("per_failure_run_signals.csv"),
675 &metrics.per_failure_run_signals,
676 )?;
677 write_dsa_metrics_csv(&run_dir.join("dsa_metrics.csv"), &prepared, &nominal, &dsa)?;
678 write_dsa_run_signals_csv(
679 &run_dir.join("dsa_run_signals.csv"),
680 &prepared,
681 &dsa.run_signals,
682 )?;
683 write_per_failure_run_dsa_signals_csv(
684 &run_dir.join("per_failure_run_dsa_signals.csv"),
685 &dsa.per_failure_run_signals,
686 )?;
687 write_lead_time_metrics_csv(
688 &run_dir.join("lead_time_metrics.csv"),
689 &metrics.per_failure_run_signals,
690 )?;
691 write_density_metrics_csv(
692 &run_dir.join("density_metrics.csv"),
693 &metrics.density_metrics,
694 )?;
695 write_trace_csvs(
696 &run_dir, &prepared, &residuals, &signs, &baselines, &grammar,
697 )?;
698 write_dsfb_signs_csv(
699 &run_dir.join("dsfb_signs.csv"),
700 &prepared,
701 &residuals,
702 &signs,
703 )?;
704 write_serialized_csv(
705 &run_dir.join("dsfb_feature_signs.csv"),
706 &scaffold_semiotics.feature_signs,
707 )?;
708 write_dsfb_motifs_csv(&run_dir.join("dsfb_motifs.csv"), &motifs)?;
709 write_dsfb_motif_labels_per_time_csv(
710 &run_dir.join("dsfb_motif_labels_per_time.csv"),
711 &prepared,
712 &motifs,
713 )?;
714 write_serialized_csv(
715 &run_dir.join("dsfb_feature_motif_timeline.csv"),
716 &scaffold_semiotics.feature_motif_timeline,
717 )?;
718 write_serialized_csv(
719 &run_dir.join("feature_motif_timeline.csv"),
720 &scaffold_semiotics.feature_motif_timeline,
721 )?;
722 write_dsfb_grammar_states_csv(
723 &run_dir.join("dsfb_grammar_states.csv"),
724 &prepared,
725 &grammar,
726 )?;
727 write_serialized_csv(
728 &run_dir.join("dsfb_feature_grammar_states.csv"),
729 &scaffold_semiotics.feature_grammar_states,
730 )?;
731 write_serialized_csv(
732 &run_dir.join("dsfb_envelope_interaction_summary.csv"),
733 &scaffold_semiotics.envelope_interaction_summary,
734 )?;
735 write_json_pretty(
736 &run_dir.join("dsfb_heuristics_bank_expanded.json"),
737 &scaffold_semiotics.heuristics_bank_expanded,
738 )?;
739 write_dsfb_semantic_matches_csv(
740 &run_dir.join("dsfb_semantic_matches.csv"),
741 &semantic_layer.semantic_matches,
742 )?;
743 write_dsfb_semantic_matches_csv(
744 &run_dir.join("dsfb_semantic_ranked_candidates.csv"),
745 &semantic_layer.ranked_candidates,
746 )?;
747 write_serialized_csv(
748 &run_dir.join("dsfb_feature_policy_decisions.csv"),
749 &scaffold_semiotics.feature_policy_decisions,
750 )?;
751 write_serialized_csv(
752 &run_dir.join("policy_decisions.csv"),
753 &scaffold_semiotics.feature_policy_decisions,
754 )?;
755 let traceability_entries = build_traceability_entries(&scaffold_semiotics);
756 write_traceability_json(
757 &run_dir.join("dsfb_traceability.json"),
758 &traceability_entries,
759 )?;
760 {
762 let manifest = DsfbRunManifest::new(
763 Utc::now().to_rfc3339(),
764 "batch_secom_no_recipe_context".to_string(),
765 traceability_entries.len(),
766 );
767 manifest.write(&run_dir.join("dsfb_run_manifest.json"))?;
768 }
769 write_json_pretty(
770 &run_dir.join("dsfb_group_definitions.json"),
771 &scaffold_semiotics.group_definitions,
772 )?;
773 write_serialized_csv(
774 &run_dir.join("dsfb_group_signs.csv"),
775 &scaffold_semiotics.group_signs,
776 )?;
777 write_serialized_csv(
778 &run_dir.join("dsfb_group_grammar_states.csv"),
779 &scaffold_semiotics.group_grammar_states,
780 )?;
781 write_serialized_csv(
782 &run_dir.join("dsfb_group_semantic_matches.csv"),
783 &scaffold_semiotics.group_semantic_matches,
784 )?;
785 write_json_pretty(
786 &run_dir.join("dsfb_structural_delta_metrics.json"),
787 &semantic_layer.structural_delta_metrics,
788 )?;
789 write_json_pretty(
790 &run_dir.join("recurrent_boundary_stats.json"),
791 &secom_addendum.recurrent_boundary_stats,
792 )?;
793 write_serialized_csv(
794 &run_dir.join("recurrent_boundary_tradeoff_curve.csv"),
795 &secom_addendum.recurrent_boundary_tradeoff_curve,
796 )?;
797 draw_recurrent_boundary_tradeoff_plot(
798 &run_dir.join("recurrent_boundary_tradeoff_plot.png"),
799 &secom_addendum.recurrent_boundary_tradeoff_curve,
800 )?;
801 write_serialized_csv(
802 &run_dir.join("dsfb_metric_regrounding.csv"),
803 &secom_addendum.metric_regrounding,
804 )?;
805 write_json_pretty(
806 &run_dir.join("target_d_regression_analysis.json"),
807 &secom_addendum.target_d_regression_analysis,
808 )?;
809 write_json_pretty(
810 &run_dir.join("missed_failure_root_cause.json"),
811 &secom_addendum.missed_failure_root_cause,
812 )?;
813 write_serialized_csv(
814 &run_dir.join("lead_time_comparison.csv"),
815 &secom_addendum.lead_time_comparison,
816 )?;
817 write_json_pretty(
818 &run_dir.join("lead_time_explanation.json"),
819 &secom_addendum.lead_time_explanation,
820 )?;
821 write_json_pretty(
822 &run_dir.join("episode_precision_metrics.json"),
823 &secom_addendum.episode_precision_metrics,
824 )?;
825 write_serialized_csv(
826 &run_dir.join("dsfb_episode_summary.csv"),
827 &[DsfbEpisodeSummaryRow {
828 raw_boundary_episodes: optimization.operator_delta_targets.baseline_episode_count,
829 dsfb_episodes: optimization.operator_delta_targets.optimized_episode_count,
830 episode_collapse_fraction: optimization.operator_delta_targets.delta_episode_count,
831 }],
832 )?;
833 write_serialized_csv(
834 &run_dir.join("dsfb_episode_precision.csv"),
835 &[DsfbEpisodePrecisionRow {
836 raw_boundary_precision: secom_addendum.episode_precision_metrics.raw_alarm_precision,
837 dsfb_precision: secom_addendum.episode_precision_metrics.dsfb_precision,
838 precision_gain_factor: secom_addendum
839 .episode_precision_metrics
840 .precision_gain_factor,
841 raw_boundary_episodes: secom_addendum.episode_precision_metrics.raw_alarm_count,
842 dsfb_episodes: secom_addendum.episode_precision_metrics.dsfb_episode_count,
843 dsfb_pre_failure_episodes: secom_addendum
844 .episode_precision_metrics
845 .dsfb_pre_failure_episode_count,
846 }],
847 )?;
848 write_serialized_csv(
849 &run_dir.join("dsfb_recall_metrics.csv"),
850 &[DsfbRecallMetricsRow {
851 detected_failures: optimization
852 .operator_delta_targets
853 .selected_configuration
854 .failure_recall,
855 total_failures: optimization
856 .operator_delta_targets
857 .selected_configuration
858 .failure_runs,
859 recall: optimization
860 .operator_delta_targets
861 .selected_configuration
862 .failure_recall as f64
863 / optimization
864 .operator_delta_targets
865 .selected_configuration
866 .failure_runs
867 .max(1) as f64,
868 }],
869 )?;
870 fs::write(
871 run_dir.join("paper_abstract_artifact.txt"),
872 &secom_addendum.paper_abstract_artifact,
873 )?;
874 let mut figures = generate_figures(
875 &run_dir, &prepared, &nominal, &residuals, &signs, &baselines, &grammar, &metrics, &dsa,
876 &config,
877 )?;
878 figures
879 .files
880 .push("dsfb_non_intrusive_architecture.png".into());
881 let report = write_reports(
882 &run_dir,
883 &config,
884 &metrics,
885 &dsa,
886 &optimization,
887 &optimization.delta_target_assessment,
888 &failure_driven,
889 &feature_cohorts,
890 &cohort_summary,
891 &rating_delta_forecast,
892 &secom_addendum,
893 &figures,
894 &heuristics,
895 &phm_support_status(data_root),
896 &secom_archive_layout,
897 )?;
898
899 let manifest_path = run_dir.join("artifact_manifest.json");
900 let metrics_path = run_dir.join("benchmark_metrics.json");
901 let phm2018_status = phm_support_status(data_root);
902 let zip_path = run_dir.join("run_bundle.zip");
903 let report_pdf_alias_path = run_dir.join("report.pdf");
904 let results_zip_path = run_dir.join("results.zip");
905 write_json_pretty(
906 &manifest_path,
907 &ArtifactManifest {
908 dataset: "SECOM".into(),
909 run_dir: run_dir.display().to_string(),
910 metrics_summary_path: metrics_path.display().to_string(),
911 baseline_comparison_summary_path: run_dir
912 .join("baseline_comparison_summary.json")
913 .display()
914 .to_string(),
915 dsa_vs_baselines_summary_path: run_dir
916 .join("dsa_vs_baselines.json")
917 .display()
918 .to_string(),
919 dsa_parameter_manifest_path: run_dir
920 .join("dsa_parameter_manifest.json")
921 .display()
922 .to_string(),
923 dsa_grid_results_path: run_dir.join("dsa_grid_results.csv").display().to_string(),
924 dsa_grid_summary_path: run_dir.join("dsa_grid_summary.json").display().to_string(),
925 dsa_feature_ranking_path: run_dir
926 .join("dsa_feature_ranking.csv")
927 .display()
928 .to_string(),
929 dsa_feature_ranking_recall_aware_path: run_dir
930 .join("dsa_feature_ranking_recall_aware.csv")
931 .display()
932 .to_string(),
933 dsa_feature_ranking_dsfb_aware_path: run_dir
934 .join("dsa_feature_ranking_dsfb_aware.csv")
935 .display()
936 .to_string(),
937 dsa_feature_ranking_burden_aware_path: run_dir
938 .join("dsa_feature_ranking_burden_aware.csv")
939 .display()
940 .to_string(),
941 dsa_feature_ranking_comparison_path: run_dir
942 .join("dsa_feature_ranking_comparison.csv")
943 .display()
944 .to_string(),
945 dsa_seed_feature_check_path: run_dir
946 .join("dsa_seed_feature_check.json")
947 .display()
948 .to_string(),
949 dsa_feature_cohorts_path: run_dir
950 .join("dsa_feature_cohorts.json")
951 .display()
952 .to_string(),
953 dsa_feature_policy_overrides_path: run_dir
954 .join("dsa_feature_policy_overrides.json")
955 .display()
956 .to_string(),
957 dsa_feature_policy_summary_path: run_dir
958 .join("dsa_feature_policy_summary.csv")
959 .display()
960 .to_string(),
961 dsa_recall_rescue_results_path: run_dir
962 .join("dsa_recall_rescue_results.csv")
963 .display()
964 .to_string(),
965 dsa_recall_critical_features_path: run_dir
966 .join("dsa_recall_critical_features.csv")
967 .display()
968 .to_string(),
969 dsa_pareto_frontier_path: run_dir
970 .join("dsa_pareto_frontier.csv")
971 .display()
972 .to_string(),
973 dsa_stage_a_candidates_path: run_dir
974 .join("dsa_stage_a_candidates.csv")
975 .display()
976 .to_string(),
977 dsa_stage_b_candidates_path: run_dir
978 .join("dsa_stage_b_candidates.csv")
979 .display()
980 .to_string(),
981 dsa_stage1_candidates_path: run_dir
982 .join("dsa_stage1_candidates.csv")
983 .display()
984 .to_string(),
985 dsa_stage2_candidates_path: run_dir
986 .join("dsa_stage2_candidates.csv")
987 .display()
988 .to_string(),
989 dsa_missed_failure_diagnostics_path: run_dir
990 .join("dsa_missed_failure_diagnostics.csv")
991 .display()
992 .to_string(),
993 dsa_delta_target_assessment_path: run_dir
994 .join("dsa_delta_target_assessment.json")
995 .display()
996 .to_string(),
997 dsa_operator_baselines_path: run_dir
998 .join("dsa_operator_baselines.json")
999 .display()
1000 .to_string(),
1001 dsa_operator_delta_targets_path: run_dir
1002 .join("dsa_operator_delta_targets.json")
1003 .display()
1004 .to_string(),
1005 dsa_operator_delta_attainment_matrix_path: run_dir
1006 .join("dsa_operator_delta_attainment_matrix.csv")
1007 .display()
1008 .to_string(),
1009 dsa_policy_operator_burden_contributions_path: run_dir
1010 .join("dsa_policy_operator_burden_contributions.csv")
1011 .display()
1012 .to_string(),
1013 dsa_recall_recovery_efficiency_path: run_dir
1014 .join("dsa_recall_recovery_efficiency.csv")
1015 .display()
1016 .to_string(),
1017 dsfb_single_change_iteration_log_path: run_dir
1018 .join("dsfb_single_change_iteration_log.csv")
1019 .display()
1020 .to_string(),
1021 optimization_log_path: run_dir.join("optimization_log.json").display().to_string(),
1022 failures_index_path: run_dir.join("failures_index.json").display().to_string(),
1023 missed_failure_priority_path: run_dir
1024 .join("missed_failure_priority.csv")
1025 .display()
1026 .to_string(),
1027 failure_case_paths: failure_driven
1028 .failure_cases
1029 .iter()
1030 .map(|case| {
1031 run_dir
1032 .join(format!("failure_case_{}.json", case.failure_id))
1033 .display()
1034 .to_string()
1035 })
1036 .collect(),
1037 feature_motif_grounding_path: run_dir
1038 .join("feature_motif_grounding.json")
1039 .display()
1040 .to_string(),
1041 feature_to_motif_path: run_dir.join("feature_to_motif.json").display().to_string(),
1042 negative_control_report_path: run_dir
1043 .join("negative_control_report.json")
1044 .display()
1045 .to_string(),
1046 dsfb_heuristics_bank_minimal_path: run_dir
1047 .join("dsfb_heuristics_bank_minimal.json")
1048 .display()
1049 .to_string(),
1050 dsfb_heuristic_provenance_path: run_dir
1051 .join("dsfb_heuristic_provenance.csv")
1052 .display()
1053 .to_string(),
1054 policy_decisions_path: run_dir.join("policy_decisions.csv").display().to_string(),
1055 policy_burden_summary_path: run_dir
1056 .join("policy_burden_summary.csv")
1057 .display()
1058 .to_string(),
1059 dsfb_feature_role_validation_path: run_dir
1060 .join("dsfb_feature_role_validation.csv")
1061 .display()
1062 .to_string(),
1063 dsfb_group_validation_path: run_dir
1064 .join("dsfb_group_validation.csv")
1065 .display()
1066 .to_string(),
1067 dsfb_vs_ewma_case_paths: failure_driven
1068 .dsfb_vs_ewma_cases
1069 .iter()
1070 .map(|case| {
1071 run_dir
1072 .join(format!("dsfb_vs_ewma_case_{}.json", case.failure_id))
1073 .display()
1074 .to_string()
1075 })
1076 .collect(),
1077 dsa_cohort_results_path: run_dir.join("dsa_cohort_results.csv").display().to_string(),
1078 dsa_cohort_results_recall_aware_path: run_dir
1079 .join("dsa_cohort_results_recall_aware.csv")
1080 .display()
1081 .to_string(),
1082 dsa_cohort_results_dsfb_aware_path: run_dir
1083 .join("dsa_cohort_results_dsfb_aware.csv")
1084 .display()
1085 .to_string(),
1086 dsa_cohort_results_burden_aware_path: run_dir
1087 .join("dsa_cohort_results_burden_aware.csv")
1088 .display()
1089 .to_string(),
1090 dsa_cohort_summary_path: run_dir
1091 .join("dsa_cohort_summary.json")
1092 .display()
1093 .to_string(),
1094 dsa_cohort_summary_recall_aware_path: run_dir
1095 .join("dsa_cohort_summary_recall_aware.json")
1096 .display()
1097 .to_string(),
1098 dsa_cohort_summary_dsfb_aware_path: run_dir
1099 .join("dsa_cohort_summary_dsfb_aware.json")
1100 .display()
1101 .to_string(),
1102 dsa_cohort_summary_burden_aware_path: run_dir
1103 .join("dsa_cohort_summary_burden_aware.json")
1104 .display()
1105 .to_string(),
1106 dsa_cohort_precursor_quality_path: run_dir
1107 .join("dsa_cohort_precursor_quality.csv")
1108 .display()
1109 .to_string(),
1110 dsa_cohort_failure_analysis_path: cohort_summary.failure_analysis.as_ref().map(|_| {
1111 run_dir
1112 .join("dsa_cohort_failure_analysis.md")
1113 .display()
1114 .to_string()
1115 }),
1116 dsa_heuristic_policy_failure_analysis_path: cohort_summary
1117 .failure_analysis
1118 .as_ref()
1119 .map(|_| {
1120 run_dir
1121 .join("dsa_heuristic_policy_failure_analysis.md")
1122 .display()
1123 .to_string()
1124 }),
1125 dsa_motif_policy_contributions_path: run_dir
1126 .join("dsa_motif_policy_contributions.csv")
1127 .display()
1128 .to_string(),
1129 dsa_policy_contribution_analysis_path: run_dir
1130 .join("dsa_policy_contribution_analysis.csv")
1131 .display()
1132 .to_string(),
1133 dsa_rating_delta_forecast_path: run_dir
1134 .join("dsa_rating_delta_forecast.json")
1135 .display()
1136 .to_string(),
1137 dsa_rating_delta_failure_analysis_path: rating_delta_failure_analysis.as_ref().map(
1138 |_| {
1139 run_dir
1140 .join("dsa_rating_delta_failure_analysis.md")
1141 .display()
1142 .to_string()
1143 },
1144 ),
1145 lead_time_metrics_path: run_dir.join("lead_time_metrics.csv").display().to_string(),
1146 density_metrics_path: run_dir.join("density_metrics.csv").display().to_string(),
1147 cusum_baseline_path: run_dir.join("cusum_baseline.csv").display().to_string(),
1148 run_energy_baseline_path: run_dir
1149 .join("run_energy_baseline.csv")
1150 .display()
1151 .to_string(),
1152 pca_fdc_baseline_path: run_dir.join("pca_fdc_baseline.csv").display().to_string(),
1153 per_failure_run_signals_path: run_dir
1154 .join("per_failure_run_signals.csv")
1155 .display()
1156 .to_string(),
1157 dsa_metrics_path: run_dir.join("dsa_metrics.csv").display().to_string(),
1158 dsa_run_signals_path: run_dir.join("dsa_run_signals.csv").display().to_string(),
1159 per_failure_run_dsa_signals_path: run_dir
1160 .join("per_failure_run_dsa_signals.csv")
1161 .display()
1162 .to_string(),
1163 dsfb_signs_path: run_dir.join("dsfb_signs.csv").display().to_string(),
1164 dsfb_feature_signs_path: run_dir.join("dsfb_feature_signs.csv").display().to_string(),
1165 dsfb_motifs_path: run_dir.join("dsfb_motifs.csv").display().to_string(),
1166 dsfb_motif_labels_per_time_path: run_dir
1167 .join("dsfb_motif_labels_per_time.csv")
1168 .display()
1169 .to_string(),
1170 dsfb_feature_motif_timeline_path: run_dir
1171 .join("dsfb_feature_motif_timeline.csv")
1172 .display()
1173 .to_string(),
1174 dsfb_grammar_states_path: run_dir
1175 .join("dsfb_grammar_states.csv")
1176 .display()
1177 .to_string(),
1178 dsfb_feature_grammar_states_path: run_dir
1179 .join("dsfb_feature_grammar_states.csv")
1180 .display()
1181 .to_string(),
1182 dsfb_envelope_interaction_summary_path: run_dir
1183 .join("dsfb_envelope_interaction_summary.csv")
1184 .display()
1185 .to_string(),
1186 dsfb_heuristics_bank_expanded_path: run_dir
1187 .join("dsfb_heuristics_bank_expanded.json")
1188 .display()
1189 .to_string(),
1190 dsfb_semantic_matches_path: run_dir
1191 .join("dsfb_semantic_matches.csv")
1192 .display()
1193 .to_string(),
1194 dsfb_semantic_ranked_candidates_path: run_dir
1195 .join("dsfb_semantic_ranked_candidates.csv")
1196 .display()
1197 .to_string(),
1198 dsfb_feature_policy_decisions_path: run_dir
1199 .join("dsfb_feature_policy_decisions.csv")
1200 .display()
1201 .to_string(),
1202 dsfb_traceability_path: run_dir
1203 .join("dsfb_traceability.json")
1204 .display()
1205 .to_string(),
1206 dsfb_group_definitions_path: run_dir
1207 .join("dsfb_group_definitions.json")
1208 .display()
1209 .to_string(),
1210 dsfb_group_signs_path: run_dir.join("dsfb_group_signs.csv").display().to_string(),
1211 dsfb_group_grammar_states_path: run_dir
1212 .join("dsfb_group_grammar_states.csv")
1213 .display()
1214 .to_string(),
1215 dsfb_group_semantic_matches_path: run_dir
1216 .join("dsfb_group_semantic_matches.csv")
1217 .display()
1218 .to_string(),
1219 dsfb_structural_delta_metrics_path: run_dir
1220 .join("dsfb_structural_delta_metrics.json")
1221 .display()
1222 .to_string(),
1223 recurrent_boundary_stats_path: run_dir
1224 .join("recurrent_boundary_stats.json")
1225 .display()
1226 .to_string(),
1227 recurrent_boundary_tradeoff_curve_path: run_dir
1228 .join("recurrent_boundary_tradeoff_curve.csv")
1229 .display()
1230 .to_string(),
1231 recurrent_boundary_tradeoff_plot_path: run_dir
1232 .join("recurrent_boundary_tradeoff_plot.png")
1233 .display()
1234 .to_string(),
1235 dsfb_metric_regrounding_path: run_dir
1236 .join("dsfb_metric_regrounding.csv")
1237 .display()
1238 .to_string(),
1239 target_d_regression_analysis_path: run_dir
1240 .join("target_d_regression_analysis.json")
1241 .display()
1242 .to_string(),
1243 missed_failure_root_cause_path: run_dir
1244 .join("missed_failure_root_cause.json")
1245 .display()
1246 .to_string(),
1247 lead_time_comparison_path: run_dir
1248 .join("lead_time_comparison.csv")
1249 .display()
1250 .to_string(),
1251 lead_time_explanation_path: run_dir
1252 .join("lead_time_explanation.json")
1253 .display()
1254 .to_string(),
1255 episode_precision_metrics_path: run_dir
1256 .join("episode_precision_metrics.json")
1257 .display()
1258 .to_string(),
1259 dsfb_episode_summary_path: run_dir
1260 .join("dsfb_episode_summary.csv")
1261 .display()
1262 .to_string(),
1263 dsfb_episode_precision_path: run_dir
1264 .join("dsfb_episode_precision.csv")
1265 .display()
1266 .to_string(),
1267 dsfb_recall_metrics_path: run_dir
1268 .join("dsfb_recall_metrics.csv")
1269 .display()
1270 .to_string(),
1271 paper_abstract_artifact_path: run_dir
1272 .join("paper_abstract_artifact.txt")
1273 .display()
1274 .to_string(),
1275 secom_archive_layout_path: run_dir
1276 .join("secom_archive_layout.json")
1277 .display()
1278 .to_string(),
1279 drsc_trace_path: figures
1280 .drsc
1281 .as_ref()
1282 .map(|drsc| run_dir.join(&drsc.trace_csv).display().to_string()),
1283 drsc_figure_path: figures.drsc.as_ref().map(|drsc| {
1284 run_dir
1285 .join("figures")
1286 .join(&drsc.figure_file)
1287 .display()
1288 .to_string()
1289 }),
1290 drsc_dsa_combined_trace_path: figures
1291 .drsc_dsa_combined
1292 .as_ref()
1293 .map(|combined| run_dir.join(&combined.trace_csv).display().to_string()),
1294 drsc_dsa_combined_figure_path: figures.drsc_dsa_combined.as_ref().map(|combined| {
1295 run_dir
1296 .join("figures")
1297 .join(&combined.figure_file)
1298 .display()
1299 .to_string()
1300 }),
1301 dsa_focus_trace_path: figures
1302 .dsa_focus
1303 .as_ref()
1304 .map(|dsa_focus| run_dir.join(&dsa_focus.trace_csv).display().to_string()),
1305 dsa_focus_figure_path: figures.dsa_focus.as_ref().map(|dsa_focus| {
1306 run_dir
1307 .join("figures")
1308 .join(&dsa_focus.figure_file)
1309 .display()
1310 .to_string()
1311 }),
1312 non_intrusive_interface_spec_path: non_intrusive_artifacts
1313 .interface_spec_path
1314 .display()
1315 .to_string(),
1316 non_intrusive_architecture_png_path: non_intrusive_artifacts
1317 .architecture_png_path
1318 .display()
1319 .to_string(),
1320 non_intrusive_architecture_svg_path: non_intrusive_artifacts
1321 .architecture_svg_path
1322 .display()
1323 .to_string(),
1324 report_markdown_path: report.markdown_path.display().to_string(),
1325 report_tex_path: report.tex_path.display().to_string(),
1326 report_pdf_path: report
1327 .pdf_path
1328 .as_ref()
1329 .map(|path| path.display().to_string()),
1330 report_pdf_alias_path: report
1331 .pdf_path
1332 .as_ref()
1333 .map(|_| report_pdf_alias_path.display().to_string()),
1334 zip_path: zip_path.display().to_string(),
1335 results_zip_path: results_zip_path.display().to_string(),
1336 },
1337 )?;
1338 if let Some(pdf_path) = &report.pdf_path {
1339 fs::copy(pdf_path, &report_pdf_alias_path)?;
1340 }
1341 zip_directory(&run_dir, &zip_path)?;
1342 fs::copy(&zip_path, &results_zip_path)?;
1343
1344 Ok(SecomRunArtifacts {
1345 run_dir,
1346 report,
1347 figures,
1348 metrics_path,
1349 manifest_path,
1350 zip_path,
1351 phm2018_status,
1352 paper_lock_metrics,
1353 })
1354}
1355
1356fn write_readme_first(run_dir: &std::path::Path, config: &crate::config::PipelineConfig) -> crate::error::Result<()> {
1361 let content = format!(
1362 r#"DSFB-SEMICONDUCTOR — RUN SUMMARY
1363=================================
1364
1365CONFIGURATION
1366 W (drift_window) : {drift_window}
1367 K (pre_failure_lookback_runs) : {lookback}
1368 tau (dsa.alert_tau) : {tau:.1}
1369 m (corroborating_feature_min) : {m}
1370 envelope_sigma : {sigma:.1}
1371 strategy : all_features [compression_biased]
1372
1373WHAT DSFB DOES
1374 - Reads residual streams produced by SECOM feature extraction (read-only).
1375 - Restructures those residuals into a compact set of structured episodes
1376 annotated with grammar states (Admissible / Boundary / Violation).
1377 - Emits advisory policy decisions (Silent / Review / Escalate) per feature.
1378 - Writes no data back to any upstream system.
1379
1380WHAT DSFB DOES NOT DO (explicit non-claims)
1381 - Does NOT alter any FDC, SPC, EWMA, or CUSUM system.
1382 - Does NOT predict failure lead-time or claim empirical lead-time advantage.
1383 - Does NOT replace process engineering review or control plan sign-off.
1384 - Does NOT claim physical attribution for any motif class observed in SECOM.
1385 - Is NOT a certified or production-qualified fault-detection system.
1386 - Removal of the DSFB layer leaves all upstream systems entirely unchanged.
1387
1388FILES IN THIS DIRECTORY
1389 metrics_summary.json — Full benchmark metric tree
1390 baseline_comparison_summary.json — DSFB vs. EWMA / CUSUM / PCA baselines
1391 dsfb_episode_precision.csv — Filtered episode count and precision
1392 dsfb_recall_metrics.csv — Failure-run recall
1393 report.tex / report.pdf — LaTeX narrative report
1394 dsfb_semiconductor_secom.zip — Full reproducibility archive
1395
1396REPRODUCIBILITY
1397 Re-run with: cargo run --release -- run-secom
1398 Paper-lock: cargo run --release -- paper-lock
1399 All outputs are deterministic under fixed config and dataset.
1400"#,
1401 drift_window = config.drift_window,
1402 lookback = config.pre_failure_lookback_runs,
1403 tau = config.dsa.alert_tau,
1404 m = config.dsa.corroborating_feature_count_min,
1405 sigma = config.envelope_sigma,
1406 );
1407 fs::write(run_dir.join("README_FIRST.txt"), content).map_err(Into::into)
1408}
1409
1410fn write_operator_summary(
1415 run_dir: &std::path::Path,
1416 config: &crate::config::PipelineConfig,
1417) -> crate::error::Result<()> {
1418 let content = format!(
1419 r#"DSFB-SEMICONDUCTOR — OPERATOR RUN SUMMARY
1420==========================================
1421
1422PURPOSE
1423 This run applies the DSFB structural semiotics observer to SECOM residuals.
1424 The observer is read-only: it does not modify any upstream control system.
1425
1426SELECTED CONFIGURATION
1427 DSA drift window : {dsa_window} (W in paper)
1428 DSA persistence runs : {dsa_persistence} (K in paper)
1429 DSA alert threshold : {dsa_tau:.1} (tau in paper)
1430 DSA corroboration count : {dsa_m} (m in paper)
1431 Feature strategy : all_features [compression_biased]
1432 Grammar drift window : {drift_window}
1433 Envelope sigma : {sigma:.1}
1434
1435WHAT THIS RUN PRODUCES
1436 - Structured episode list (Watch / Review / Escalate per feature)
1437 - Traceability chain: dsfb_traceability.json
1438 - Benchmark metrics: benchmark_metrics.json
1439 - Full report: report.tex / report.pdf
1440
1441WHAT THIS RUN DOES NOT PRODUCE
1442 - No modifications to SPC, EWMA, FDC, or CUSUM thresholds
1443 - No write-back to upstream control systems
1444 - No physical attribution claims
1445 - No predictive lead-time guarantee
1446
1447REPRODUCTION
1448 cargo run --release --bin dsfb-semiconductor -- run-secom
1449 cargo run --release --bin dsfb-semiconductor -- paper-lock
1450
1451All outputs are deterministic under fixed configuration and dataset.
1452"#,
1453 dsa_window = config.dsa.window,
1454 dsa_persistence = config.dsa.persistence_runs,
1455 dsa_tau = config.dsa.alert_tau,
1456 dsa_m = config.dsa.corroborating_feature_count_min,
1457 drift_window = config.drift_window,
1458 sigma = config.envelope_sigma,
1459 );
1460 fs::write(run_dir.join("RUN_SUMMARY_OPERATOR.txt"), content).map_err(Into::into)
1461}
1462
1463fn build_baseline_comparison_summary(
1464 metrics: &BenchmarkMetrics,
1465 dsa: &DsaEvaluation,
1466 secom_archive_layout: &SecomArchiveLayout,
1467 config: &PipelineConfig,
1468) -> BaselineComparisonSummary {
1469 BaselineComparisonSummary {
1470 dataset: "SECOM".into(),
1471 secom_archive_layout_note: secom_archive_layout.note.clone(),
1472 feature_count_used_by_crate: metrics.summary.dataset_summary.feature_count,
1473 failure_runs: metrics.summary.failure_runs,
1474 analyzable_feature_count: metrics.summary.analyzable_feature_count,
1475 grammar_imputation_suppression_points: metrics
1476 .summary
1477 .grammar_imputation_suppression_points,
1478 lookback_runs: config.pre_failure_lookback_runs,
1479 failure_run_recall: FailureRunRecallSummary {
1480 dsfb_raw_signal: metrics.summary.failure_runs_with_preceding_dsfb_raw_signal,
1481 dsfb_persistent_signal: metrics
1482 .summary
1483 .failure_runs_with_preceding_dsfb_persistent_signal,
1484 dsfb_raw_boundary_signal: metrics
1485 .summary
1486 .failure_runs_with_preceding_dsfb_raw_boundary_signal,
1487 dsfb_persistent_boundary_signal: metrics
1488 .summary
1489 .failure_runs_with_preceding_dsfb_persistent_boundary_signal,
1490 dsfb_raw_violation_signal: metrics
1491 .summary
1492 .failure_runs_with_preceding_dsfb_raw_violation_signal,
1493 dsfb_persistent_violation_signal: metrics
1494 .summary
1495 .failure_runs_with_preceding_dsfb_persistent_violation_signal,
1496 dsfb_dsa_signal: dsa.summary.failure_run_recall,
1497 ewma_signal: metrics.summary.failure_runs_with_preceding_ewma_signal,
1498 cusum_signal: metrics.summary.failure_runs_with_preceding_cusum_signal,
1499 run_energy_signal: metrics
1500 .summary
1501 .failure_runs_with_preceding_run_energy_signal,
1502 pca_fdc_signal: metrics.summary.failure_runs_with_preceding_pca_fdc_signal,
1503 threshold_signal: metrics.summary.failure_runs_with_preceding_threshold_signal,
1504 },
1505 pass_run_nuisance_proxy: PassRunNuisanceSummary {
1506 dsfb_raw_boundary_signal_runs: metrics.summary.pass_runs_with_dsfb_raw_boundary_signal,
1507 dsfb_persistent_boundary_signal_runs: metrics
1508 .summary
1509 .pass_runs_with_dsfb_persistent_boundary_signal,
1510 dsfb_raw_violation_signal_runs: metrics
1511 .summary
1512 .pass_runs_with_dsfb_raw_violation_signal,
1513 dsfb_persistent_violation_signal_runs: metrics
1514 .summary
1515 .pass_runs_with_dsfb_persistent_violation_signal,
1516 dsfb_dsa_signal_runs: (dsa.summary.pass_run_nuisance_proxy
1517 * metrics.summary.pass_runs as f64)
1518 .round() as usize,
1519 ewma_signal_runs: metrics.summary.pass_runs_with_ewma_signal,
1520 cusum_signal_runs: metrics.summary.pass_runs_with_cusum_signal,
1521 run_energy_signal_runs: metrics.summary.pass_runs_with_run_energy_signal,
1522 pca_fdc_signal_runs: metrics.summary.pass_runs_with_pca_fdc_signal,
1523 threshold_signal_runs: metrics.summary.pass_runs_with_threshold_signal,
1524 dsfb_raw_boundary_signal_rate: metrics.summary.pass_run_dsfb_raw_boundary_nuisance_rate,
1525 dsfb_persistent_boundary_signal_rate: metrics
1526 .summary
1527 .pass_run_dsfb_persistent_boundary_nuisance_rate,
1528 dsfb_raw_violation_signal_rate: metrics
1529 .summary
1530 .pass_run_dsfb_raw_violation_nuisance_rate,
1531 dsfb_persistent_violation_signal_rate: metrics
1532 .summary
1533 .pass_run_dsfb_persistent_violation_nuisance_rate,
1534 dsfb_dsa_signal_rate: dsa.summary.pass_run_nuisance_proxy,
1535 ewma_signal_rate: metrics.summary.pass_run_ewma_nuisance_rate,
1536 cusum_signal_rate: metrics.summary.pass_run_cusum_nuisance_rate,
1537 run_energy_signal_rate: metrics.summary.pass_run_run_energy_nuisance_rate,
1538 pca_fdc_signal_rate: metrics.summary.pass_run_pca_fdc_nuisance_rate,
1539 threshold_signal_rate: metrics.summary.pass_run_threshold_nuisance_rate,
1540 },
1541 lead_time_summary: metrics.lead_time_summary.clone(),
1542 density_summary: metrics.density_summary.clone(),
1543 boundary_episode_summary: metrics.boundary_episode_summary.clone(),
1544 dsa_comparison_summary: Some(dsa.comparison_summary.clone()),
1545 }
1546}
1547
1548fn write_json_pretty<T: Serialize>(path: &Path, value: &T) -> Result<()> {
1549 let json = serde_json::to_string_pretty(value)?;
1550 fs::write(path, json)?;
1551 Ok(())
1552}
1553
1554fn write_feature_metrics_csv(path: &Path, metrics: &BenchmarkMetrics) -> Result<()> {
1555 let mut writer = csv::Writer::from_path(path)?;
1556 for feature in &metrics.feature_metrics {
1557 writer.serialize(feature)?;
1558 }
1559 writer.flush()?;
1560 Ok(())
1561}
1562
1563fn write_per_failure_run_signals_csv(path: &Path, records: &[PerFailureRunSignal]) -> Result<()> {
1564 let mut writer = csv::Writer::from_path(path)?;
1565 for record in records {
1566 writer.serialize(record)?;
1567 }
1568 writer.flush()?;
1569 Ok(())
1570}
1571
1572fn write_dsa_metrics_csv(
1573 path: &Path,
1574 prepared: &crate::preprocessing::PreparedDataset,
1575 nominal: &crate::nominal::NominalModel,
1576 dsa: &DsaEvaluation,
1577) -> Result<()> {
1578 let mut writer = csv::Writer::from_path(path)?;
1579 writer.write_record([
1580 "feature_index",
1581 "feature_name",
1582 "run_index",
1583 "timestamp",
1584 "label",
1585 "boundary_basis_hit",
1586 "drift_outward_hit",
1587 "slew_hit",
1588 "motif_hit",
1589 "boundary_density_W",
1590 "drift_persistence_W",
1591 "slew_density_W",
1592 "ewma_occupancy_W",
1593 "motif_recurrence_W",
1594 "fragmentation_proxy_W",
1595 "consistent",
1596 "dsa_score",
1597 "dsa_active",
1598 "numeric_dsa_alert",
1599 "dsa_alert",
1600 "resolved_alert_class",
1601 "policy_state",
1602 "policy_suppressed_to_silent",
1603 "rescue_transition",
1604 "rescued_to_review",
1605 ])?;
1606
1607 for trace in &dsa.traces {
1608 if !nominal.features[trace.feature_index].analyzable {
1609 continue;
1610 }
1611 for run_index in 0..trace.dsa_score.len() {
1612 writer.write_record([
1613 trace.feature_index.to_string(),
1614 trace.feature_name.clone(),
1615 run_index.to_string(),
1616 prepared.timestamps[run_index]
1617 .format("%Y-%m-%d %H:%M:%S")
1618 .to_string(),
1619 prepared.labels[run_index].to_string(),
1620 trace.boundary_basis_hit[run_index].to_string(),
1621 trace.drift_outward_hit[run_index].to_string(),
1622 trace.slew_hit[run_index].to_string(),
1623 trace.motif_hit[run_index].to_string(),
1624 trace.boundary_density_w[run_index].to_string(),
1625 trace.drift_persistence_w[run_index].to_string(),
1626 trace.slew_density_w[run_index].to_string(),
1627 trace.ewma_occupancy_w[run_index].to_string(),
1628 trace.motif_recurrence_w[run_index].to_string(),
1629 trace.fragmentation_proxy_w[run_index].to_string(),
1630 trace.consistent[run_index].to_string(),
1631 trace.dsa_score[run_index].to_string(),
1632 trace.dsa_active[run_index].to_string(),
1633 trace.numeric_dsa_alert[run_index].to_string(),
1634 trace.dsa_alert[run_index].to_string(),
1635 format!("{:?}", trace.resolved_alert_class[run_index]),
1636 trace.policy_state[run_index].as_lowercase().to_string(),
1637 trace.policy_suppressed_to_silent[run_index].to_string(),
1638 trace.rescue_transition[run_index].clone(),
1639 trace.rescued_to_review[run_index].to_string(),
1640 ])?;
1641 }
1642 }
1643
1644 writer.flush()?;
1645 Ok(())
1646}
1647
1648fn write_dsa_run_signals_csv(
1649 path: &Path,
1650 prepared: &crate::preprocessing::PreparedDataset,
1651 run_signals: &DsaRunSignals,
1652) -> Result<()> {
1653 let mut writer = csv::Writer::from_path(path)?;
1654 writer.write_record([
1655 "run_index",
1656 "timestamp",
1657 "label",
1658 "primary_run_signal",
1659 "corroborating_feature_count_min",
1660 "primary_run_alert",
1661 "any_feature_dsa_alert",
1662 "any_feature_raw_violation",
1663 "feature_count_dsa_alert",
1664 "watch_feature_count",
1665 "review_feature_count",
1666 "escalate_feature_count",
1667 "strict_escalate_run_alert",
1668 "numeric_primary_run_alert",
1669 "numeric_feature_count_dsa_alert",
1670 ])?;
1671
1672 for run_index in 0..prepared.labels.len() {
1673 writer.write_record([
1674 run_index.to_string(),
1675 prepared.timestamps[run_index]
1676 .format("%Y-%m-%d %H:%M:%S")
1677 .to_string(),
1678 prepared.labels[run_index].to_string(),
1679 run_signals.primary_run_signal.clone(),
1680 run_signals.corroborating_feature_count_min.to_string(),
1681 run_signals.primary_run_alert[run_index].to_string(),
1682 run_signals.any_feature_dsa_alert[run_index].to_string(),
1683 run_signals.any_feature_raw_violation[run_index].to_string(),
1684 run_signals.feature_count_dsa_alert[run_index].to_string(),
1685 run_signals.watch_feature_count[run_index].to_string(),
1686 run_signals.review_feature_count[run_index].to_string(),
1687 run_signals.escalate_feature_count[run_index].to_string(),
1688 run_signals.strict_escalate_run_alert[run_index].to_string(),
1689 run_signals.numeric_primary_run_alert[run_index].to_string(),
1690 run_signals.numeric_feature_count_dsa_alert[run_index].to_string(),
1691 ])?;
1692 }
1693
1694 writer.flush()?;
1695 Ok(())
1696}
1697
1698fn write_per_failure_run_dsa_signals_csv(
1699 path: &Path,
1700 records: &[PerFailureRunDsaSignal],
1701) -> Result<()> {
1702 let mut writer = csv::Writer::from_path(path)?;
1703 writer.write_record([
1704 "failure_run_index",
1705 "failure_timestamp",
1706 "earliest_dsa_run",
1707 "earliest_primary_source",
1708 "earliest_dsa_feature_index",
1709 "earliest_dsa_feature_name",
1710 "dsa_lead_runs",
1711 "threshold_lead_runs",
1712 "ewma_lead_runs",
1713 "cusum_lead_runs",
1714 "run_energy_lead_runs",
1715 "pca_fdc_lead_runs",
1716 "dsa_minus_cusum_delta_runs",
1717 "dsa_minus_run_energy_delta_runs",
1718 "dsa_minus_pca_fdc_delta_runs",
1719 "dsa_minus_threshold_delta_runs",
1720 "dsa_minus_ewma_delta_runs",
1721 "dsa_alerting_feature_count",
1722 "max_dsa_score_in_lookback",
1723 "max_dsa_score_feature_index",
1724 "max_dsa_score_feature_name",
1725 "max_dsa_score_run_index",
1726 "max_dsa_score_boundary_density_w",
1727 "max_dsa_score_drift_persistence_w",
1728 "max_dsa_score_slew_density_w",
1729 "max_dsa_score_ewma_occupancy_w",
1730 "max_dsa_score_motif_recurrence_w",
1731 "max_dsa_score_fragmentation_proxy_w",
1732 "max_dsa_score_consistent",
1733 "max_dsa_score_policy_state",
1734 "max_dsa_score_resolved_alert_class",
1735 "max_dsa_score_numeric_dsa_alert",
1736 "max_dsa_score_dsa_alert",
1737 "max_dsa_score_policy_suppressed",
1738 "max_dsa_score_rescue_transition",
1739 ])?;
1740 for record in records {
1741 writer.write_record([
1742 record.failure_run_index.to_string(),
1743 record.failure_timestamp.clone(),
1744 option_to_string(record.earliest_dsa_run),
1745 record.earliest_primary_source.clone().unwrap_or_default(),
1746 option_to_string(record.earliest_dsa_feature_index),
1747 record.earliest_dsa_feature_name.clone().unwrap_or_default(),
1748 option_to_string(record.dsa_lead_runs),
1749 option_to_string(record.threshold_lead_runs),
1750 option_to_string(record.ewma_lead_runs),
1751 option_to_string(record.cusum_lead_runs),
1752 option_to_string(record.run_energy_lead_runs),
1753 option_to_string(record.pca_fdc_lead_runs),
1754 option_to_string(record.dsa_minus_cusum_delta_runs),
1755 option_to_string(record.dsa_minus_run_energy_delta_runs),
1756 option_to_string(record.dsa_minus_pca_fdc_delta_runs),
1757 option_to_string(record.dsa_minus_threshold_delta_runs),
1758 option_to_string(record.dsa_minus_ewma_delta_runs),
1759 record.dsa_alerting_feature_count.to_string(),
1760 option_to_string(record.max_dsa_score_in_lookback),
1761 option_to_string(record.max_dsa_score_feature_index),
1762 record
1763 .max_dsa_score_feature_name
1764 .clone()
1765 .unwrap_or_default(),
1766 option_to_string(record.max_dsa_score_run_index),
1767 option_to_string(record.max_dsa_score_boundary_density_w),
1768 option_to_string(record.max_dsa_score_drift_persistence_w),
1769 option_to_string(record.max_dsa_score_slew_density_w),
1770 option_to_string(record.max_dsa_score_ewma_occupancy_w),
1771 option_to_string(record.max_dsa_score_motif_recurrence_w),
1772 option_to_string(record.max_dsa_score_fragmentation_proxy_w),
1773 option_to_string(record.max_dsa_score_consistent),
1774 record
1775 .max_dsa_score_policy_state
1776 .clone()
1777 .unwrap_or_default(),
1778 record
1779 .max_dsa_score_resolved_alert_class
1780 .clone()
1781 .unwrap_or_default(),
1782 option_to_string(record.max_dsa_score_numeric_dsa_alert),
1783 option_to_string(record.max_dsa_score_dsa_alert),
1784 option_to_string(record.max_dsa_score_policy_suppressed),
1785 record
1786 .max_dsa_score_rescue_transition
1787 .clone()
1788 .unwrap_or_default(),
1789 ])?;
1790 }
1791 writer.flush()?;
1792 Ok(())
1793}
1794
1795fn write_lead_time_metrics_csv(path: &Path, records: &[PerFailureRunSignal]) -> Result<()> {
1796 let mut writer = csv::Writer::from_path(path)?;
1797 writer.write_record([
1798 "failure_run_index",
1799 "failure_timestamp",
1800 "earliest_dsfb_raw_boundary_run",
1801 "earliest_dsfb_persistent_boundary_run",
1802 "earliest_dsfb_raw_violation_run",
1803 "earliest_dsfb_persistent_violation_run",
1804 "earliest_threshold_run",
1805 "earliest_ewma_run",
1806 "earliest_cusum_run",
1807 "earliest_run_energy_run",
1808 "earliest_pca_fdc_run",
1809 "dsfb_raw_boundary_lead_runs",
1810 "dsfb_persistent_boundary_lead_runs",
1811 "dsfb_raw_violation_lead_runs",
1812 "dsfb_persistent_violation_lead_runs",
1813 "threshold_lead_runs",
1814 "ewma_lead_runs",
1815 "cusum_lead_runs",
1816 "run_energy_lead_runs",
1817 "pca_fdc_lead_runs",
1818 "dsfb_raw_boundary_minus_cusum_delta_runs",
1819 "dsfb_raw_boundary_minus_run_energy_delta_runs",
1820 "dsfb_raw_boundary_minus_pca_fdc_delta_runs",
1821 "dsfb_raw_boundary_minus_threshold_delta_runs",
1822 "dsfb_raw_boundary_minus_ewma_delta_runs",
1823 "dsfb_persistent_boundary_minus_cusum_delta_runs",
1824 "dsfb_persistent_boundary_minus_run_energy_delta_runs",
1825 "dsfb_persistent_boundary_minus_pca_fdc_delta_runs",
1826 "dsfb_persistent_boundary_minus_threshold_delta_runs",
1827 "dsfb_persistent_boundary_minus_ewma_delta_runs",
1828 "dsfb_raw_violation_minus_cusum_delta_runs",
1829 "dsfb_raw_violation_minus_run_energy_delta_runs",
1830 "dsfb_raw_violation_minus_pca_fdc_delta_runs",
1831 "dsfb_raw_violation_minus_threshold_delta_runs",
1832 "dsfb_raw_violation_minus_ewma_delta_runs",
1833 "dsfb_persistent_violation_minus_cusum_delta_runs",
1834 "dsfb_persistent_violation_minus_run_energy_delta_runs",
1835 "dsfb_persistent_violation_minus_pca_fdc_delta_runs",
1836 "dsfb_persistent_violation_minus_threshold_delta_runs",
1837 "dsfb_persistent_violation_minus_ewma_delta_runs",
1838 ])?;
1839
1840 for record in records {
1841 writer.write_record([
1842 record.failure_run_index.to_string(),
1843 record.failure_timestamp.clone(),
1844 option_to_string(record.earliest_dsfb_raw_boundary_run),
1845 option_to_string(record.earliest_dsfb_persistent_boundary_run),
1846 option_to_string(record.earliest_dsfb_raw_violation_run),
1847 option_to_string(record.earliest_dsfb_persistent_violation_run),
1848 option_to_string(record.earliest_threshold_run),
1849 option_to_string(record.earliest_ewma_run),
1850 option_to_string(record.earliest_cusum_run),
1851 option_to_string(record.earliest_run_energy_run),
1852 option_to_string(record.earliest_pca_fdc_run),
1853 option_to_string(record.dsfb_raw_boundary_lead_runs),
1854 option_to_string(record.dsfb_persistent_boundary_lead_runs),
1855 option_to_string(record.dsfb_raw_violation_lead_runs),
1856 option_to_string(record.dsfb_persistent_violation_lead_runs),
1857 option_to_string(record.threshold_lead_runs),
1858 option_to_string(record.ewma_lead_runs),
1859 option_to_string(record.cusum_lead_runs),
1860 option_to_string(record.run_energy_lead_runs),
1861 option_to_string(record.pca_fdc_lead_runs),
1862 option_to_string(record.dsfb_raw_boundary_minus_cusum_delta_runs),
1863 option_to_string(record.dsfb_raw_boundary_minus_run_energy_delta_runs),
1864 option_to_string(record.dsfb_raw_boundary_minus_pca_fdc_delta_runs),
1865 option_to_string(record.dsfb_raw_boundary_minus_threshold_delta_runs),
1866 option_to_string(record.dsfb_raw_boundary_minus_ewma_delta_runs),
1867 option_to_string(record.dsfb_persistent_boundary_minus_cusum_delta_runs),
1868 option_to_string(record.dsfb_persistent_boundary_minus_run_energy_delta_runs),
1869 option_to_string(record.dsfb_persistent_boundary_minus_pca_fdc_delta_runs),
1870 option_to_string(record.dsfb_persistent_boundary_minus_threshold_delta_runs),
1871 option_to_string(record.dsfb_persistent_boundary_minus_ewma_delta_runs),
1872 option_to_string(record.dsfb_raw_violation_minus_cusum_delta_runs),
1873 option_to_string(record.dsfb_raw_violation_minus_run_energy_delta_runs),
1874 option_to_string(record.dsfb_raw_violation_minus_pca_fdc_delta_runs),
1875 option_to_string(record.dsfb_raw_violation_minus_threshold_delta_runs),
1876 option_to_string(record.dsfb_raw_violation_minus_ewma_delta_runs),
1877 option_to_string(record.dsfb_persistent_violation_minus_cusum_delta_runs),
1878 option_to_string(record.dsfb_persistent_violation_minus_run_energy_delta_runs),
1879 option_to_string(record.dsfb_persistent_violation_minus_pca_fdc_delta_runs),
1880 option_to_string(record.dsfb_persistent_violation_minus_threshold_delta_runs),
1881 option_to_string(record.dsfb_persistent_violation_minus_ewma_delta_runs),
1882 ])?;
1883 }
1884
1885 writer.flush()?;
1886 Ok(())
1887}
1888
1889fn write_density_metrics_csv(path: &Path, records: &[DensityMetricRecord]) -> Result<()> {
1890 let mut writer = csv::Writer::from_path(path)?;
1891 for record in records {
1892 writer.serialize(record)?;
1893 }
1894 writer.flush()?;
1895 Ok(())
1896}
1897
1898fn write_dsfb_signs_csv(
1899 path: &Path,
1900 prepared: &crate::preprocessing::PreparedDataset,
1901 residuals: &crate::residual::ResidualSet,
1902 signs: &crate::signs::SignSet,
1903) -> Result<()> {
1904 let mut writer = csv::Writer::from_path(path)?;
1905 writer.write_record([
1906 "feature_index",
1907 "feature_name",
1908 "run_index",
1909 "timestamp",
1910 "label",
1911 "residual",
1912 "drift",
1913 "slew",
1914 "residual_norm",
1915 "is_imputed",
1916 "drift_threshold",
1917 "slew_threshold",
1918 ])?;
1919 for (residual_trace, sign_trace) in residuals.traces.iter().zip(&signs.traces) {
1920 for run_index in 0..residual_trace.residuals.len() {
1921 writer.write_record([
1922 residual_trace.feature_index.to_string(),
1923 residual_trace.feature_name.clone(),
1924 run_index.to_string(),
1925 prepared.timestamps[run_index]
1926 .format("%Y-%m-%d %H:%M:%S")
1927 .to_string(),
1928 prepared.labels[run_index].to_string(),
1929 residual_trace.residuals[run_index].to_string(),
1930 sign_trace.drift[run_index].to_string(),
1931 sign_trace.slew[run_index].to_string(),
1932 residual_trace.norms[run_index].to_string(),
1933 residual_trace.is_imputed[run_index].to_string(),
1934 sign_trace.drift_threshold.to_string(),
1935 sign_trace.slew_threshold.to_string(),
1936 ])?;
1937 }
1938 }
1939 writer.flush()?;
1940 Ok(())
1941}
1942
1943fn write_dsfb_motifs_csv(path: &Path, motifs: &crate::semiotics::MotifSet) -> Result<()> {
1944 let mut writer = csv::Writer::from_path(path)?;
1945 for row in &motifs.summary_rows {
1946 writer.serialize(row)?;
1947 }
1948 writer.flush()?;
1949 Ok(())
1950}
1951
1952fn write_dsfb_motif_labels_per_time_csv(
1953 path: &Path,
1954 prepared: &crate::preprocessing::PreparedDataset,
1955 motifs: &crate::semiotics::MotifSet,
1956) -> Result<()> {
1957 let mut writer = csv::Writer::from_path(path)?;
1958 writer.write_record([
1959 "feature_index",
1960 "feature_name",
1961 "run_index",
1962 "timestamp",
1963 "label",
1964 "motif_label",
1965 ])?;
1966 for trace in &motifs.traces {
1967 for (run_index, motif_label) in trace.labels.iter().enumerate() {
1968 writer.write_record([
1969 trace.feature_index.to_string(),
1970 trace.feature_name.clone(),
1971 run_index.to_string(),
1972 prepared.timestamps[run_index]
1973 .format("%Y-%m-%d %H:%M:%S")
1974 .to_string(),
1975 prepared.labels[run_index].to_string(),
1976 motif_label.as_lowercase().to_string(),
1977 ])?;
1978 }
1979 }
1980 writer.flush()?;
1981 Ok(())
1982}
1983
1984fn write_dsfb_grammar_states_csv(
1985 path: &Path,
1986 prepared: &crate::preprocessing::PreparedDataset,
1987 grammar: &crate::grammar::GrammarSet,
1988) -> Result<()> {
1989 let mut writer = csv::Writer::from_path(path)?;
1990 writer.write_record([
1991 "feature_index",
1992 "feature_name",
1993 "run_index",
1994 "timestamp",
1995 "label",
1996 "raw_state",
1997 "confirmed_state",
1998 "persistent_boundary",
1999 "persistent_violation",
2000 "suppressed_by_imputation",
2001 "raw_reason",
2002 "confirmed_reason",
2003 ])?;
2004 for trace in &grammar.traces {
2005 for run_index in 0..trace.raw_states.len() {
2006 writer.write_record([
2007 trace.feature_index.to_string(),
2008 trace.feature_name.clone(),
2009 run_index.to_string(),
2010 prepared.timestamps[run_index]
2011 .format("%Y-%m-%d %H:%M:%S")
2012 .to_string(),
2013 prepared.labels[run_index].to_string(),
2014 format!("{:?}", trace.raw_states[run_index]),
2015 format!("{:?}", trace.states[run_index]),
2016 trace.persistent_boundary[run_index].to_string(),
2017 trace.persistent_violation[run_index].to_string(),
2018 trace.suppressed_by_imputation[run_index].to_string(),
2019 format!("{:?}", trace.raw_reasons[run_index]),
2020 format!("{:?}", trace.reasons[run_index]),
2021 ])?;
2022 }
2023 }
2024 writer.flush()?;
2025 Ok(())
2026}
2027
2028fn write_dsfb_semantic_matches_csv(
2029 path: &Path,
2030 rows: &[crate::semiotics::SemanticMatchRecord],
2031) -> Result<()> {
2032 let mut writer = csv::Writer::from_path(path)?;
2033 for row in rows {
2034 writer.serialize(row)?;
2035 }
2036 writer.flush()?;
2037 Ok(())
2038}
2039
2040fn write_serialized_csv<T: Serialize>(path: &Path, rows: &[T]) -> Result<()> {
2041 let mut writer = csv::Writer::from_path(path)?;
2042 for row in rows {
2043 writer.serialize(row)?;
2044 }
2045 writer.flush()?;
2046 Ok(())
2047}
2048
2049fn write_trace_csvs(
2050 run_dir: &Path,
2051 prepared: &crate::preprocessing::PreparedDataset,
2052 residuals: &crate::residual::ResidualSet,
2053 signs: &crate::signs::SignSet,
2054 baselines: &crate::baselines::BaselineSet,
2055 grammar: &crate::grammar::GrammarSet,
2056) -> Result<()> {
2057 let mut residual_writer = csv::Writer::from_path(run_dir.join("residuals.csv"))?;
2058 let mut drift_writer = csv::Writer::from_path(run_dir.join("drifts.csv"))?;
2059 let mut slew_writer = csv::Writer::from_path(run_dir.join("slews.csv"))?;
2060 let mut ewma_writer = csv::Writer::from_path(run_dir.join("ewma_baseline.csv"))?;
2061 let mut cusum_writer = csv::Writer::from_path(run_dir.join("cusum_baseline.csv"))?;
2062 let mut run_energy_writer = csv::Writer::from_path(run_dir.join("run_energy_baseline.csv"))?;
2063 let mut pca_fdc_writer = csv::Writer::from_path(run_dir.join("pca_fdc_baseline.csv"))?;
2064 let mut grammar_writer = csv::Writer::from_path(run_dir.join("grammar_states.csv"))?;
2065
2066 residual_writer.write_record([
2067 "run_index",
2068 "timestamp",
2069 "label",
2070 "feature",
2071 "imputed_value",
2072 "is_imputed",
2073 "residual",
2074 "residual_norm",
2075 "threshold_alarm",
2076 ])?;
2077 drift_writer.write_record(["run_index", "timestamp", "feature", "drift"])?;
2078 slew_writer.write_record(["run_index", "timestamp", "feature", "slew"])?;
2079 ewma_writer.write_record([
2080 "run_index",
2081 "timestamp",
2082 "feature",
2083 "ewma",
2084 "healthy_mean",
2085 "healthy_std",
2086 "threshold",
2087 "alarm",
2088 ])?;
2089 cusum_writer.write_record([
2090 "run_index",
2091 "timestamp",
2092 "feature",
2093 "cusum",
2094 "healthy_mean",
2095 "healthy_std",
2096 "kappa",
2097 "alarm_threshold",
2098 "alarm",
2099 ])?;
2100 run_energy_writer.write_record([
2101 "run_index",
2102 "timestamp",
2103 "label",
2104 "run_energy",
2105 "healthy_mean",
2106 "healthy_std",
2107 "threshold",
2108 "analyzable_feature_count",
2109 "alarm",
2110 ])?;
2111 pca_fdc_writer.write_record([
2112 "run_index",
2113 "timestamp",
2114 "label",
2115 "pca_t2",
2116 "pca_t2_healthy_mean",
2117 "pca_t2_healthy_std",
2118 "pca_t2_threshold",
2119 "pca_spe",
2120 "pca_spe_healthy_mean",
2121 "pca_spe_healthy_std",
2122 "pca_spe_threshold",
2123 "retained_components",
2124 "explained_variance_fraction",
2125 "target_variance_explained",
2126 "analyzable_feature_count",
2127 "alarm",
2128 ])?;
2129 grammar_writer.write_record([
2130 "run_index",
2131 "timestamp",
2132 "feature",
2133 "raw_state",
2134 "confirmed_state",
2135 "persistent_boundary",
2136 "persistent_violation",
2137 "suppressed_by_imputation",
2138 "raw_reason",
2139 "confirmed_reason",
2140 ])?;
2141
2142 for feature_index in 0..residuals.traces.len() {
2143 let residual_trace = &residuals.traces[feature_index];
2144 let sign_trace = &signs.traces[feature_index];
2145 let ewma_trace = &baselines.ewma[feature_index];
2146 let cusum_trace = &baselines.cusum[feature_index];
2147 let grammar_trace = &grammar.traces[feature_index];
2148 for run_index in 0..prepared.timestamps.len() {
2149 let timestamp = prepared.timestamps[run_index]
2150 .format("%Y-%m-%d %H:%M:%S")
2151 .to_string();
2152 residual_writer.write_record([
2153 run_index.to_string(),
2154 timestamp.clone(),
2155 prepared.labels[run_index].to_string(),
2156 residual_trace.feature_name.clone(),
2157 residual_trace.imputed_values[run_index].to_string(),
2158 residual_trace.is_imputed[run_index].to_string(),
2159 residual_trace.residuals[run_index].to_string(),
2160 residual_trace.norms[run_index].to_string(),
2161 residual_trace.threshold_alarm[run_index].to_string(),
2162 ])?;
2163 drift_writer.write_record([
2164 run_index.to_string(),
2165 timestamp.clone(),
2166 residual_trace.feature_name.clone(),
2167 sign_trace.drift[run_index].to_string(),
2168 ])?;
2169 slew_writer.write_record([
2170 run_index.to_string(),
2171 timestamp.clone(),
2172 residual_trace.feature_name.clone(),
2173 sign_trace.slew[run_index].to_string(),
2174 ])?;
2175 ewma_writer.write_record([
2176 run_index.to_string(),
2177 timestamp.clone(),
2178 residual_trace.feature_name.clone(),
2179 ewma_trace.ewma[run_index].to_string(),
2180 ewma_trace.healthy_mean.to_string(),
2181 ewma_trace.healthy_std.to_string(),
2182 ewma_trace.threshold.to_string(),
2183 ewma_trace.alarm[run_index].to_string(),
2184 ])?;
2185 cusum_writer.write_record([
2186 run_index.to_string(),
2187 timestamp.clone(),
2188 residual_trace.feature_name.clone(),
2189 cusum_trace.cusum[run_index].to_string(),
2190 cusum_trace.healthy_mean.to_string(),
2191 cusum_trace.healthy_std.to_string(),
2192 cusum_trace.kappa.to_string(),
2193 cusum_trace.alarm_threshold.to_string(),
2194 cusum_trace.alarm[run_index].to_string(),
2195 ])?;
2196 grammar_writer.write_record([
2197 run_index.to_string(),
2198 timestamp,
2199 residual_trace.feature_name.clone(),
2200 format!("{:?}", grammar_trace.raw_states[run_index]),
2201 format!("{:?}", grammar_trace.states[run_index]),
2202 grammar_trace.persistent_boundary[run_index].to_string(),
2203 grammar_trace.persistent_violation[run_index].to_string(),
2204 grammar_trace.suppressed_by_imputation[run_index].to_string(),
2205 format!("{:?}", grammar_trace.raw_reasons[run_index]),
2206 format!("{:?}", grammar_trace.reasons[run_index]),
2207 ])?;
2208 }
2209 }
2210
2211 for run_index in 0..prepared.timestamps.len() {
2212 run_energy_writer.write_record([
2213 run_index.to_string(),
2214 prepared.timestamps[run_index]
2215 .format("%Y-%m-%d %H:%M:%S")
2216 .to_string(),
2217 prepared.labels[run_index].to_string(),
2218 baselines.run_energy.energy[run_index].to_string(),
2219 baselines.run_energy.healthy_mean.to_string(),
2220 baselines.run_energy.healthy_std.to_string(),
2221 baselines.run_energy.threshold.to_string(),
2222 baselines.run_energy.analyzable_feature_count.to_string(),
2223 baselines.run_energy.alarm[run_index].to_string(),
2224 ])?;
2225 pca_fdc_writer.write_record([
2226 run_index.to_string(),
2227 prepared.timestamps[run_index]
2228 .format("%Y-%m-%d %H:%M:%S")
2229 .to_string(),
2230 prepared.labels[run_index].to_string(),
2231 baselines.pca_fdc.t2[run_index].to_string(),
2232 baselines.pca_fdc.t2_healthy_mean.to_string(),
2233 baselines.pca_fdc.t2_healthy_std.to_string(),
2234 baselines.pca_fdc.t2_threshold.to_string(),
2235 baselines.pca_fdc.spe[run_index].to_string(),
2236 baselines.pca_fdc.spe_healthy_mean.to_string(),
2237 baselines.pca_fdc.spe_healthy_std.to_string(),
2238 baselines.pca_fdc.spe_threshold.to_string(),
2239 baselines.pca_fdc.retained_components.to_string(),
2240 baselines.pca_fdc.explained_variance_fraction.to_string(),
2241 baselines.pca_fdc.target_variance_explained.to_string(),
2242 baselines.pca_fdc.analyzable_feature_count.to_string(),
2243 baselines.pca_fdc.alarm[run_index].to_string(),
2244 ])?;
2245 }
2246
2247 residual_writer.flush()?;
2248 drift_writer.flush()?;
2249 slew_writer.flush()?;
2250 ewma_writer.flush()?;
2251 cusum_writer.flush()?;
2252 run_energy_writer.flush()?;
2253 pca_fdc_writer.flush()?;
2254 grammar_writer.flush()?;
2255 Ok(())
2256}
2257
2258fn option_to_string<T: ToString>(value: Option<T>) -> String {
2259 value.map(|value| value.to_string()).unwrap_or_default()
2260}
2261
2262fn zip_directory(run_dir: &Path, zip_path: &Path) -> Result<()> {
2263 let file = File::create(zip_path)?;
2264 let mut zip = zip::ZipWriter::new(file);
2265 let options = SimpleFileOptions::default()
2266 .compression_method(zip::CompressionMethod::Deflated)
2267 .unix_permissions(0o644);
2268 add_directory_contents(&mut zip, run_dir, run_dir, zip_path, options)?;
2269 zip.finish()?;
2270 Ok(())
2271}
2272
2273fn add_directory_contents(
2274 zip: &mut zip::ZipWriter<File>,
2275 root: &Path,
2276 current: &Path,
2277 zip_path: &Path,
2278 options: SimpleFileOptions,
2279) -> Result<()> {
2280 for entry in fs::read_dir(current)? {
2281 let entry = entry?;
2282 let path = entry.path();
2283 if path == zip_path {
2284 continue;
2285 }
2286 if path.is_dir() {
2287 add_directory_contents(zip, root, &path, zip_path, options)?;
2288 } else {
2289 let relative = path
2290 .strip_prefix(root)
2291 .map_err(|err| DsfbSemiconductorError::DatasetFormat(err.to_string()))?;
2292 zip.start_file(relative.to_string_lossy().replace('\\', "/"), options)?;
2293 let bytes = fs::read(&path)?;
2294 zip.write_all(&bytes)?;
2295 }
2296 }
2297 Ok(())
2298}