use std::sync::Mutex;
use rusqlite::Connection;
use crate::error::FetchiumError;
use crate::intelligence::enable_wal;
#[derive(Debug, Clone, serde::Serialize)]
pub struct CalibratedConfidence {
pub stated: f64,
pub calibrated: f64,
pub sample_size: u64,
pub domain_category: String,
pub has_calibration_data: bool,
}
impl CalibratedConfidence {
pub fn display(&self) -> String {
if self.has_calibration_data {
format!(
"{:.0}% (calibrated: {:.0}%, n={})",
self.stated * 100.0,
self.calibrated * 100.0,
self.sample_size,
)
} else {
format!("{:.0}% (uncalibrated)", self.stated * 100.0)
}
}
}
pub struct ConfidenceCalibrationEngine {
conn: Mutex<Connection>,
}
impl ConfidenceCalibrationEngine {
pub fn new(db_path: &std::path::Path) -> Result<Self, FetchiumError> {
let conn = Connection::open(db_path)?;
enable_wal(&conn)?;
conn.execute_batch(
"CREATE TABLE IF NOT EXISTS predictions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
domain_category TEXT NOT NULL,
stated_confidence REAL NOT NULL,
actually_correct INTEGER, -- NULL until verified; 0 or 1
claim TEXT NOT NULL,
source_url TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
verified_at TEXT
);
CREATE TABLE IF NOT EXISTS calibration_bins (
domain_category TEXT NOT NULL,
bin_lower REAL NOT NULL,
bin_upper REAL NOT NULL,
total_count INTEGER NOT NULL DEFAULT 0,
correct_count INTEGER NOT NULL DEFAULT 0,
actual_accuracy REAL,
PRIMARY KEY (domain_category, bin_lower)
);",
)?;
Ok(Self {
conn: Mutex::new(conn),
})
}
pub fn record_prediction(
&self,
domain_category: &str,
stated_confidence: f64,
claim: &str,
source_url: Option<&str>,
) -> Result<u64, FetchiumError> {
let conn = self.conn.lock().unwrap();
conn.execute(
"INSERT INTO predictions (domain_category, stated_confidence, claim, source_url)
VALUES (?1, ?2, ?3, ?4)",
rusqlite::params![domain_category, stated_confidence, claim, source_url],
)?;
Ok(conn.last_insert_rowid() as u64)
}
pub fn verify_prediction(
&self,
prediction_id: u64,
actually_correct: bool,
) -> Result<(), FetchiumError> {
let conn = self.conn.lock().unwrap();
conn.execute(
"UPDATE predictions SET
actually_correct = ?2,
verified_at = datetime('now')
WHERE id = ?1",
rusqlite::params![prediction_id as i64, actually_correct as i32],
)?;
let (domain_category, confidence): (String, f64) = conn.query_row(
"SELECT domain_category, stated_confidence FROM predictions WHERE id = ?1",
[prediction_id as i64],
|row| Ok((row.get(0)?, row.get(1)?)),
)?;
let bin_lower = (confidence * 10.0).floor() / 10.0;
let bin_upper = (bin_lower + 0.1).min(1.0);
let correct_int = actually_correct as i32;
conn.execute(
"INSERT INTO calibration_bins
(domain_category, bin_lower, bin_upper, total_count, correct_count, actual_accuracy)
VALUES (?1, ?2, ?3, 1, ?4, CAST(?4 AS REAL))
ON CONFLICT(domain_category, bin_lower) DO UPDATE SET
total_count = total_count + 1,
correct_count = correct_count + ?4,
actual_accuracy = CAST(correct_count + ?4 AS REAL) / (total_count + 1)",
rusqlite::params![domain_category, bin_lower, bin_upper, correct_int],
)?;
Ok(())
}
pub fn calibrate(
&self,
domain_category: &str,
stated_confidence: f64,
) -> Result<CalibratedConfidence, FetchiumError> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT bin_lower, actual_accuracy, total_count
FROM calibration_bins
WHERE domain_category = ?1 AND total_count >= 10
ORDER BY bin_lower",
)?;
let bins: Vec<(f64, f64, i64)> = stmt
.query_map([domain_category], |row| {
Ok((row.get(0)?, row.get(1)?, row.get(2)?))
})?
.filter_map(|r| r.ok())
.collect();
if bins.is_empty() {
return Ok(CalibratedConfidence {
stated: stated_confidence,
calibrated: stated_confidence,
sample_size: 0,
domain_category: domain_category.to_string(),
has_calibration_data: false,
});
}
let calibrated = isotonic_interpolate(&bins, stated_confidence);
let total_samples: i64 = bins.iter().map(|(_, _, n)| n).sum();
Ok(CalibratedConfidence {
stated: stated_confidence,
calibrated,
sample_size: total_samples as u64,
domain_category: domain_category.to_string(),
has_calibration_data: true,
})
}
pub fn calibration_summary(&self) -> Result<Vec<(String, u64, u64)>, FetchiumError> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT domain_category, COUNT(*), SUM(total_count)
FROM calibration_bins
GROUP BY domain_category
ORDER BY SUM(total_count) DESC",
)?;
let results = stmt
.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, i64>(1)? as u64,
row.get::<_, i64>(2)? as u64,
))
})?
.filter_map(|r| r.ok())
.collect();
Ok(results)
}
}
fn isotonic_interpolate(bins: &[(f64, f64, i64)], target: f64) -> f64 {
if bins.is_empty() {
return target;
}
let mut below: Option<(f64, f64)> = None;
let mut above: Option<(f64, f64)> = None;
for &(lower, accuracy, _) in bins {
if lower <= target {
below = Some((lower, accuracy));
}
if lower > target && above.is_none() {
above = Some((lower, accuracy));
}
}
match (below, above) {
(Some((l, la)), Some((u, ua))) => {
let range = u - l;
if range < 1e-9 {
return la;
}
let t = (target - l) / range;
(la + t * (ua - la)).clamp(0.0, 1.0)
}
(Some((_, la)), None) => la,
(None, Some((_, ua))) => ua,
(None, None) => target,
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::NamedTempFile;
#[test]
fn no_calibration_data_returns_stated() {
let tmp = NamedTempFile::new().unwrap();
let cce = ConfidenceCalibrationEngine::new(tmp.path()).unwrap();
let cal = cce.calibrate("tech", 0.85).unwrap();
assert!(!cal.has_calibration_data);
assert!((cal.calibrated - 0.85).abs() < 1e-9);
}
#[test]
fn calibrated_after_10_verified_predictions() {
let tmp = NamedTempFile::new().unwrap();
let cce = ConfidenceCalibrationEngine::new(tmp.path()).unwrap();
let ids: Vec<u64> = (0..12)
.map(|i| {
cce.record_prediction("tech", 0.85, &format!("claim {i}"), None)
.unwrap()
})
.collect();
for (i, id) in ids.iter().enumerate() {
cce.verify_prediction(*id, i < 9).unwrap();
}
let cal = cce.calibrate("tech", 0.85).unwrap();
assert!(cal.has_calibration_data, "should have calibration data");
assert!(
cal.calibrated < 0.85,
"calibrated={:.2} should be below stated 0.85",
cal.calibrated
);
}
#[test]
fn isotonic_interpolate_between_bins() {
let bins = vec![(0.7_f64, 0.65_f64, 20), (0.8_f64, 0.80_f64, 20)];
let result = isotonic_interpolate(&bins, 0.75);
assert!((result - 0.725).abs() < 0.01, "result={result:.3}");
}
}