fetchium_core/intelligence/
cce.rs1use std::sync::Mutex;
9
10use rusqlite::Connection;
11
12use crate::error::FetchiumError;
13use crate::intelligence::enable_wal;
14
15#[derive(Debug, Clone, serde::Serialize)]
17pub struct CalibratedConfidence {
18 pub stated: f64,
20 pub calibrated: f64,
22 pub sample_size: u64,
24 pub domain_category: String,
25 pub has_calibration_data: bool,
26}
27
28impl CalibratedConfidence {
29 pub fn display(&self) -> String {
31 if self.has_calibration_data {
32 format!(
33 "{:.0}% (calibrated: {:.0}%, n={})",
34 self.stated * 100.0,
35 self.calibrated * 100.0,
36 self.sample_size,
37 )
38 } else {
39 format!("{:.0}% (uncalibrated)", self.stated * 100.0)
40 }
41 }
42}
43
44pub struct ConfidenceCalibrationEngine {
46 conn: Mutex<Connection>,
47}
48
49impl ConfidenceCalibrationEngine {
50 pub fn new(db_path: &std::path::Path) -> Result<Self, FetchiumError> {
51 let conn = Connection::open(db_path)?;
52 enable_wal(&conn)?;
53 conn.execute_batch(
54 "CREATE TABLE IF NOT EXISTS predictions (
55 id INTEGER PRIMARY KEY AUTOINCREMENT,
56 domain_category TEXT NOT NULL,
57 stated_confidence REAL NOT NULL,
58 actually_correct INTEGER, -- NULL until verified; 0 or 1
59 claim TEXT NOT NULL,
60 source_url TEXT,
61 created_at TEXT NOT NULL DEFAULT (datetime('now')),
62 verified_at TEXT
63 );
64 CREATE TABLE IF NOT EXISTS calibration_bins (
65 domain_category TEXT NOT NULL,
66 bin_lower REAL NOT NULL,
67 bin_upper REAL NOT NULL,
68 total_count INTEGER NOT NULL DEFAULT 0,
69 correct_count INTEGER NOT NULL DEFAULT 0,
70 actual_accuracy REAL,
71 PRIMARY KEY (domain_category, bin_lower)
72 );",
73 )?;
74 Ok(Self {
75 conn: Mutex::new(conn),
76 })
77 }
78
79 pub fn record_prediction(
81 &self,
82 domain_category: &str,
83 stated_confidence: f64,
84 claim: &str,
85 source_url: Option<&str>,
86 ) -> Result<u64, FetchiumError> {
87 let conn = self.conn.lock().unwrap();
88 conn.execute(
89 "INSERT INTO predictions (domain_category, stated_confidence, claim, source_url)
90 VALUES (?1, ?2, ?3, ?4)",
91 rusqlite::params![domain_category, stated_confidence, claim, source_url],
92 )?;
93 Ok(conn.last_insert_rowid() as u64)
94 }
95
96 pub fn verify_prediction(
98 &self,
99 prediction_id: u64,
100 actually_correct: bool,
101 ) -> Result<(), FetchiumError> {
102 let conn = self.conn.lock().unwrap();
103
104 conn.execute(
105 "UPDATE predictions SET
106 actually_correct = ?2,
107 verified_at = datetime('now')
108 WHERE id = ?1",
109 rusqlite::params![prediction_id as i64, actually_correct as i32],
110 )?;
111
112 let (domain_category, confidence): (String, f64) = conn.query_row(
113 "SELECT domain_category, stated_confidence FROM predictions WHERE id = ?1",
114 [prediction_id as i64],
115 |row| Ok((row.get(0)?, row.get(1)?)),
116 )?;
117
118 let bin_lower = (confidence * 10.0).floor() / 10.0;
119 let bin_upper = (bin_lower + 0.1).min(1.0);
120 let correct_int = actually_correct as i32;
121
122 conn.execute(
123 "INSERT INTO calibration_bins
124 (domain_category, bin_lower, bin_upper, total_count, correct_count, actual_accuracy)
125 VALUES (?1, ?2, ?3, 1, ?4, CAST(?4 AS REAL))
126 ON CONFLICT(domain_category, bin_lower) DO UPDATE SET
127 total_count = total_count + 1,
128 correct_count = correct_count + ?4,
129 actual_accuracy = CAST(correct_count + ?4 AS REAL) / (total_count + 1)",
130 rusqlite::params![domain_category, bin_lower, bin_upper, correct_int],
131 )?;
132
133 Ok(())
134 }
135
136 pub fn calibrate(
141 &self,
142 domain_category: &str,
143 stated_confidence: f64,
144 ) -> Result<CalibratedConfidence, FetchiumError> {
145 let conn = self.conn.lock().unwrap();
146
147 let mut stmt = conn.prepare(
148 "SELECT bin_lower, actual_accuracy, total_count
149 FROM calibration_bins
150 WHERE domain_category = ?1 AND total_count >= 10
151 ORDER BY bin_lower",
152 )?;
153
154 let bins: Vec<(f64, f64, i64)> = stmt
155 .query_map([domain_category], |row| {
156 Ok((row.get(0)?, row.get(1)?, row.get(2)?))
157 })?
158 .filter_map(|r| r.ok())
159 .collect();
160
161 if bins.is_empty() {
162 return Ok(CalibratedConfidence {
163 stated: stated_confidence,
164 calibrated: stated_confidence,
165 sample_size: 0,
166 domain_category: domain_category.to_string(),
167 has_calibration_data: false,
168 });
169 }
170
171 let calibrated = isotonic_interpolate(&bins, stated_confidence);
172 let total_samples: i64 = bins.iter().map(|(_, _, n)| n).sum();
173
174 Ok(CalibratedConfidence {
175 stated: stated_confidence,
176 calibrated,
177 sample_size: total_samples as u64,
178 domain_category: domain_category.to_string(),
179 has_calibration_data: true,
180 })
181 }
182
183 pub fn calibration_summary(&self) -> Result<Vec<(String, u64, u64)>, FetchiumError> {
185 let conn = self.conn.lock().unwrap();
186 let mut stmt = conn.prepare(
187 "SELECT domain_category, COUNT(*), SUM(total_count)
188 FROM calibration_bins
189 GROUP BY domain_category
190 ORDER BY SUM(total_count) DESC",
191 )?;
192 let results = stmt
193 .query_map([], |row| {
194 Ok((
195 row.get::<_, String>(0)?,
196 row.get::<_, i64>(1)? as u64,
197 row.get::<_, i64>(2)? as u64,
198 ))
199 })?
200 .filter_map(|r| r.ok())
201 .collect();
202 Ok(results)
203 }
204}
205
206fn isotonic_interpolate(bins: &[(f64, f64, i64)], target: f64) -> f64 {
208 if bins.is_empty() {
209 return target;
210 }
211
212 let mut below: Option<(f64, f64)> = None;
213 let mut above: Option<(f64, f64)> = None;
214
215 for &(lower, accuracy, _) in bins {
216 if lower <= target {
217 below = Some((lower, accuracy));
218 }
219 if lower > target && above.is_none() {
220 above = Some((lower, accuracy));
221 }
222 }
223
224 match (below, above) {
225 (Some((l, la)), Some((u, ua))) => {
226 let range = u - l;
227 if range < 1e-9 {
228 return la;
229 }
230 let t = (target - l) / range;
231 (la + t * (ua - la)).clamp(0.0, 1.0)
232 }
233 (Some((_, la)), None) => la,
234 (None, Some((_, ua))) => ua,
235 (None, None) => target,
236 }
237}
238
239#[cfg(test)]
240mod tests {
241 use super::*;
242 use tempfile::NamedTempFile;
243
244 #[test]
245 fn no_calibration_data_returns_stated() {
246 let tmp = NamedTempFile::new().unwrap();
247 let cce = ConfidenceCalibrationEngine::new(tmp.path()).unwrap();
248 let cal = cce.calibrate("tech", 0.85).unwrap();
249 assert!(!cal.has_calibration_data);
250 assert!((cal.calibrated - 0.85).abs() < 1e-9);
251 }
252
253 #[test]
254 fn calibrated_after_10_verified_predictions() {
255 let tmp = NamedTempFile::new().unwrap();
256 let cce = ConfidenceCalibrationEngine::new(tmp.path()).unwrap();
257
258 let ids: Vec<u64> = (0..12)
260 .map(|i| {
261 cce.record_prediction("tech", 0.85, &format!("claim {i}"), None)
262 .unwrap()
263 })
264 .collect();
265 for (i, id) in ids.iter().enumerate() {
267 cce.verify_prediction(*id, i < 9).unwrap();
268 }
269 let cal = cce.calibrate("tech", 0.85).unwrap();
270 assert!(cal.has_calibration_data, "should have calibration data");
271 assert!(
272 cal.calibrated < 0.85,
273 "calibrated={:.2} should be below stated 0.85",
274 cal.calibrated
275 );
276 }
277
278 #[test]
279 fn isotonic_interpolate_between_bins() {
280 let bins = vec![(0.7_f64, 0.65_f64, 20), (0.8_f64, 0.80_f64, 20)];
281 let result = isotonic_interpolate(&bins, 0.75);
282 assert!((result - 0.725).abs() < 0.01, "result={result:.3}");
283 }
284}