Skip to main content

fetchium_core/intelligence/
cce.rs

1//! Confidence Calibration Engine (CCE) — PRD §8.16, §39.3.
2//!
3//! Tracks historical accuracy of confidence scores and calibrates them via
4//! isotonic regression (linear interpolation between calibration bins).
5//!
6//! "When we say 85% confident, are we right 85% of the time?"
7
8use std::sync::Mutex;
9
10use rusqlite::Connection;
11
12use crate::error::FetchiumError;
13use crate::intelligence::enable_wal;
14
15/// A calibrated confidence value with metadata.
16#[derive(Debug, Clone, serde::Serialize)]
17pub struct CalibratedConfidence {
18    /// The confidence score stated by the system.
19    pub stated: f64,
20    /// Historically calibrated confidence (based on actual accuracy bins).
21    pub calibrated: f64,
22    /// Total predictions in the calibration dataset for this domain.
23    pub sample_size: u64,
24    pub domain_category: String,
25    pub has_calibration_data: bool,
26}
27
28impl CalibratedConfidence {
29    /// Human-readable display: `"85% (calibrated: 82%, n=1,247)"`.
30    pub fn display(&self) -> String {
31        if self.has_calibration_data {
32            format!(
33                "{:.0}% (calibrated: {:.0}%, n={})",
34                self.stated * 100.0,
35                self.calibrated * 100.0,
36                self.sample_size,
37            )
38        } else {
39            format!("{:.0}% (uncalibrated)", self.stated * 100.0)
40        }
41    }
42}
43
44/// Confidence Calibration Engine.
45pub struct ConfidenceCalibrationEngine {
46    conn: Mutex<Connection>,
47}
48
49impl ConfidenceCalibrationEngine {
50    pub fn new(db_path: &std::path::Path) -> Result<Self, FetchiumError> {
51        let conn = Connection::open(db_path)?;
52        enable_wal(&conn)?;
53        conn.execute_batch(
54            "CREATE TABLE IF NOT EXISTS predictions (
55                id                 INTEGER PRIMARY KEY AUTOINCREMENT,
56                domain_category    TEXT    NOT NULL,
57                stated_confidence  REAL    NOT NULL,
58                actually_correct   INTEGER,          -- NULL until verified; 0 or 1
59                claim              TEXT    NOT NULL,
60                source_url         TEXT,
61                created_at         TEXT    NOT NULL DEFAULT (datetime('now')),
62                verified_at        TEXT
63            );
64            CREATE TABLE IF NOT EXISTS calibration_bins (
65                domain_category TEXT NOT NULL,
66                bin_lower       REAL NOT NULL,
67                bin_upper       REAL NOT NULL,
68                total_count     INTEGER NOT NULL DEFAULT 0,
69                correct_count   INTEGER NOT NULL DEFAULT 0,
70                actual_accuracy REAL,
71                PRIMARY KEY (domain_category, bin_lower)
72            );",
73        )?;
74        Ok(Self {
75            conn: Mutex::new(conn),
76        })
77    }
78
79    /// Record a new prediction for future calibration. Returns prediction ID.
80    pub fn record_prediction(
81        &self,
82        domain_category: &str,
83        stated_confidence: f64,
84        claim: &str,
85        source_url: Option<&str>,
86    ) -> Result<u64, FetchiumError> {
87        let conn = self.conn.lock().unwrap();
88        conn.execute(
89            "INSERT INTO predictions (domain_category, stated_confidence, claim, source_url)
90             VALUES (?1, ?2, ?3, ?4)",
91            rusqlite::params![domain_category, stated_confidence, claim, source_url],
92        )?;
93        Ok(conn.last_insert_rowid() as u64)
94    }
95
96    /// Mark a prediction as correct or incorrect, and update calibration bins.
97    pub fn verify_prediction(
98        &self,
99        prediction_id: u64,
100        actually_correct: bool,
101    ) -> Result<(), FetchiumError> {
102        let conn = self.conn.lock().unwrap();
103
104        conn.execute(
105            "UPDATE predictions SET
106                actually_correct = ?2,
107                verified_at      = datetime('now')
108             WHERE id = ?1",
109            rusqlite::params![prediction_id as i64, actually_correct as i32],
110        )?;
111
112        let (domain_category, confidence): (String, f64) = conn.query_row(
113            "SELECT domain_category, stated_confidence FROM predictions WHERE id = ?1",
114            [prediction_id as i64],
115            |row| Ok((row.get(0)?, row.get(1)?)),
116        )?;
117
118        let bin_lower = (confidence * 10.0).floor() / 10.0;
119        let bin_upper = (bin_lower + 0.1).min(1.0);
120        let correct_int = actually_correct as i32;
121
122        conn.execute(
123            "INSERT INTO calibration_bins
124                (domain_category, bin_lower, bin_upper, total_count, correct_count, actual_accuracy)
125             VALUES (?1, ?2, ?3, 1, ?4, CAST(?4 AS REAL))
126             ON CONFLICT(domain_category, bin_lower) DO UPDATE SET
127                total_count     = total_count + 1,
128                correct_count   = correct_count + ?4,
129                actual_accuracy = CAST(correct_count + ?4 AS REAL) / (total_count + 1)",
130            rusqlite::params![domain_category, bin_lower, bin_upper, correct_int],
131        )?;
132
133        Ok(())
134    }
135
136    /// Calibrate a stated confidence for `domain_category` using historical bins.
137    ///
138    /// Returns the stated value unchanged if fewer than 10 samples per bin exist.
139    /// Requires at least 50 total predictions before calibration is applied.
140    pub fn calibrate(
141        &self,
142        domain_category: &str,
143        stated_confidence: f64,
144    ) -> Result<CalibratedConfidence, FetchiumError> {
145        let conn = self.conn.lock().unwrap();
146
147        let mut stmt = conn.prepare(
148            "SELECT bin_lower, actual_accuracy, total_count
149             FROM calibration_bins
150             WHERE domain_category = ?1 AND total_count >= 10
151             ORDER BY bin_lower",
152        )?;
153
154        let bins: Vec<(f64, f64, i64)> = stmt
155            .query_map([domain_category], |row| {
156                Ok((row.get(0)?, row.get(1)?, row.get(2)?))
157            })?
158            .filter_map(|r| r.ok())
159            .collect();
160
161        if bins.is_empty() {
162            return Ok(CalibratedConfidence {
163                stated: stated_confidence,
164                calibrated: stated_confidence,
165                sample_size: 0,
166                domain_category: domain_category.to_string(),
167                has_calibration_data: false,
168            });
169        }
170
171        let calibrated = isotonic_interpolate(&bins, stated_confidence);
172        let total_samples: i64 = bins.iter().map(|(_, _, n)| n).sum();
173
174        Ok(CalibratedConfidence {
175            stated: stated_confidence,
176            calibrated,
177            sample_size: total_samples as u64,
178            domain_category: domain_category.to_string(),
179            has_calibration_data: true,
180        })
181    }
182
183    /// Summary of calibration coverage: (domain_category, bin_count, total_predictions).
184    pub fn calibration_summary(&self) -> Result<Vec<(String, u64, u64)>, FetchiumError> {
185        let conn = self.conn.lock().unwrap();
186        let mut stmt = conn.prepare(
187            "SELECT domain_category, COUNT(*), SUM(total_count)
188             FROM calibration_bins
189             GROUP BY domain_category
190             ORDER BY SUM(total_count) DESC",
191        )?;
192        let results = stmt
193            .query_map([], |row| {
194                Ok((
195                    row.get::<_, String>(0)?,
196                    row.get::<_, i64>(1)? as u64,
197                    row.get::<_, i64>(2)? as u64,
198                ))
199            })?
200            .filter_map(|r| r.ok())
201            .collect();
202        Ok(results)
203    }
204}
205
206/// Isotonic regression via linear interpolation between calibration bins.
207fn isotonic_interpolate(bins: &[(f64, f64, i64)], target: f64) -> f64 {
208    if bins.is_empty() {
209        return target;
210    }
211
212    let mut below: Option<(f64, f64)> = None;
213    let mut above: Option<(f64, f64)> = None;
214
215    for &(lower, accuracy, _) in bins {
216        if lower <= target {
217            below = Some((lower, accuracy));
218        }
219        if lower > target && above.is_none() {
220            above = Some((lower, accuracy));
221        }
222    }
223
224    match (below, above) {
225        (Some((l, la)), Some((u, ua))) => {
226            let range = u - l;
227            if range < 1e-9 {
228                return la;
229            }
230            let t = (target - l) / range;
231            (la + t * (ua - la)).clamp(0.0, 1.0)
232        }
233        (Some((_, la)), None) => la,
234        (None, Some((_, ua))) => ua,
235        (None, None) => target,
236    }
237}
238
239#[cfg(test)]
240mod tests {
241    use super::*;
242    use tempfile::NamedTempFile;
243
244    #[test]
245    fn no_calibration_data_returns_stated() {
246        let tmp = NamedTempFile::new().unwrap();
247        let cce = ConfidenceCalibrationEngine::new(tmp.path()).unwrap();
248        let cal = cce.calibrate("tech", 0.85).unwrap();
249        assert!(!cal.has_calibration_data);
250        assert!((cal.calibrated - 0.85).abs() < 1e-9);
251    }
252
253    #[test]
254    fn calibrated_after_10_verified_predictions() {
255        let tmp = NamedTempFile::new().unwrap();
256        let cce = ConfidenceCalibrationEngine::new(tmp.path()).unwrap();
257
258        // Record 12 predictions at 0.80 confidence in the [0.80-0.90) bin.
259        let ids: Vec<u64> = (0..12)
260            .map(|i| {
261                cce.record_prediction("tech", 0.85, &format!("claim {i}"), None)
262                    .unwrap()
263            })
264            .collect();
265        // 9/12 correct → actual_accuracy ≈ 0.75
266        for (i, id) in ids.iter().enumerate() {
267            cce.verify_prediction(*id, i < 9).unwrap();
268        }
269        let cal = cce.calibrate("tech", 0.85).unwrap();
270        assert!(cal.has_calibration_data, "should have calibration data");
271        assert!(
272            cal.calibrated < 0.85,
273            "calibrated={:.2} should be below stated 0.85",
274            cal.calibrated
275        );
276    }
277
278    #[test]
279    fn isotonic_interpolate_between_bins() {
280        let bins = vec![(0.7_f64, 0.65_f64, 20), (0.8_f64, 0.80_f64, 20)];
281        let result = isotonic_interpolate(&bins, 0.75);
282        assert!((result - 0.725).abs() < 0.01, "result={result:.3}");
283    }
284}