Skip to main content

fetchium_core/evolve/
retrain.rs

1//! CEP retrain report generation (PRD §39).
2//!
3//! Analyses the CEP ML predictor's performance over recent extractions
4//! and generates a report recommending whether retraining is needed.
5
6use crate::error::FetchiumResult;
7use serde::{Deserialize, Serialize};
8
9/// A data point for the retrain analysis.
10#[derive(Debug, Clone, Serialize, Deserialize)]
11pub struct ExtractionRecord {
12    pub url: String,
13    /// Layer predicted by the ML predictor.
14    pub predicted_layer: u8,
15    /// Layer actually used after escalation.
16    pub actual_layer: u8,
17    /// Token count extracted.
18    pub tokens_extracted: usize,
19    pub success: bool,
20    pub recorded_at: String,
21}
22
23/// Report summarising whether the CEP predictor needs retraining.
24#[derive(Debug, Clone, Serialize, Deserialize)]
25pub struct RetrainReport {
26    pub total_samples: usize,
27    pub correct_predictions: usize,
28    pub accuracy: f64,
29    pub layer_confusion: Vec<LayerConfusion>,
30    pub recommendation: String,
31    pub generated_at: String,
32}
33
34/// Number of times predicted layer X was actually Y.
35#[derive(Debug, Clone, Serialize, Deserialize)]
36pub struct LayerConfusion {
37    pub predicted: u8,
38    pub actual: u8,
39    pub count: usize,
40}
41
42impl RetrainReport {
43    /// Build a retrain report from a set of extraction records.
44    pub fn from_records(records: &[ExtractionRecord]) -> FetchiumResult<Self> {
45        let total = records.len();
46        if total == 0 {
47            return Ok(Self {
48                total_samples: 0,
49                correct_predictions: 0,
50                accuracy: 1.0,
51                layer_confusion: vec![],
52                recommendation: "No data — retrain not applicable.".into(),
53                generated_at: chrono::Utc::now().to_rfc3339(),
54            });
55        }
56
57        let correct = records
58            .iter()
59            .filter(|r| r.predicted_layer == r.actual_layer)
60            .count();
61        let accuracy = correct as f64 / total as f64;
62
63        // Build confusion counts
64        let mut conf: std::collections::HashMap<(u8, u8), usize> = std::collections::HashMap::new();
65        for r in records {
66            if r.predicted_layer != r.actual_layer {
67                *conf.entry((r.predicted_layer, r.actual_layer)).or_insert(0) += 1;
68            }
69        }
70        let mut layer_confusion: Vec<LayerConfusion> = conf
71            .into_iter()
72            .map(|((p, a), c)| LayerConfusion {
73                predicted: p,
74                actual: a,
75                count: c,
76            })
77            .collect();
78        layer_confusion.sort_by(|a, b| b.count.cmp(&a.count));
79
80        let recommendation = if accuracy >= 0.90 {
81            format!(
82                "Predictor healthy ({:.1}% accuracy). No retrain needed.",
83                accuracy * 100.0
84            )
85        } else if accuracy >= 0.75 {
86            format!(
87                "Moderate drift ({:.1}% accuracy). Consider retraining when >1000 new samples collected.",
88                accuracy * 100.0
89            )
90        } else {
91            format!(
92                "Significant drift ({:.1}% accuracy). Retrain recommended immediately.",
93                accuracy * 100.0
94            )
95        };
96
97        Ok(Self {
98            total_samples: total,
99            correct_predictions: correct,
100            accuracy,
101            layer_confusion,
102            recommendation,
103            generated_at: chrono::Utc::now().to_rfc3339(),
104        })
105    }
106
107    pub fn to_markdown(&self) -> String {
108        let mut md = format!(
109            "# CEP Predictor Retrain Report\n\n\
110             | Metric | Value |\n|--------|-------|\n\
111             | Samples | {} |\n\
112             | Correct | {} |\n\
113             | Accuracy | {:.1}% |\n\
114             | Generated | {} |\n\n",
115            self.total_samples,
116            self.correct_predictions,
117            self.accuracy * 100.0,
118            self.generated_at,
119        );
120        md.push_str(&format!("**Recommendation:** {}\n\n", self.recommendation));
121        if !self.layer_confusion.is_empty() {
122            md.push_str("## Top Layer Confusions\n\n| Predicted | Actual | Count |\n|-----------|--------|-------|\n");
123            for lc in self.layer_confusion.iter().take(10) {
124                md.push_str(&format!(
125                    "| L{} | L{} | {} |\n",
126                    lc.predicted, lc.actual, lc.count
127                ));
128            }
129        }
130        md
131    }
132}
133
134#[cfg(test)]
135mod tests {
136    use super::*;
137
138    fn make_record(pred: u8, actual: u8) -> ExtractionRecord {
139        ExtractionRecord {
140            url: "https://example.com".into(),
141            predicted_layer: pred,
142            actual_layer: actual,
143            tokens_extracted: 500,
144            success: true,
145            recorded_at: chrono::Utc::now().to_rfc3339(),
146        }
147    }
148
149    #[test]
150    fn test_perfect_accuracy() {
151        let records: Vec<_> = (0..20).map(|_| make_record(1, 1)).collect();
152        let report = RetrainReport::from_records(&records).unwrap();
153        assert!((report.accuracy - 1.0).abs() < 1e-9);
154        assert!(report.recommendation.contains("healthy"));
155    }
156
157    #[test]
158    fn test_low_accuracy_triggers_retrain() {
159        let mut records: Vec<ExtractionRecord> = (0..10).map(|_| make_record(1, 1)).collect();
160        records.extend((0..90).map(|_| make_record(1, 3)));
161        let report = RetrainReport::from_records(&records).unwrap();
162        assert!(report.accuracy < 0.75);
163        assert!(report.recommendation.contains("immediately"));
164    }
165
166    #[test]
167    fn test_empty_records() {
168        let report = RetrainReport::from_records(&[]).unwrap();
169        assert_eq!(report.total_samples, 0);
170    }
171}