Skip to main content

fetchium_core/intelligence/
edf.rs

1//! Evidence Decay Function (EDF) — PRD §8.14.
2//!
3//! Claims decay in reliability based on their content domain.
4//! Formula: decayed = base_confidence × e^(−λ × age_days)
5//! where λ = ln(2) / half_life_days.
6//!
7//! Self-calibrating: adjusts half-lives when staleness flags turn out to be wrong.
8
9use std::collections::HashMap;
10
11use rusqlite::Connection;
12
13use crate::error::FetchiumError;
14use crate::intelligence::enable_wal;
15
16// ─── Domain half-lives (in days) ─────────────────────────────────────────────
17
18/// Default half-lives for each content domain category.
19static DEFAULT_HALF_LIVES: &[(&str, f64)] = &[
20    ("ai_ml_benchmarks", 90.0),    // 3 months — model scores change rapidly
21    ("tech_news", 14.0),           // 2 weeks
22    ("medical_trials", 730.0),     // 2 years
23    ("legal_precedent", 3650.0),   // 10 years
24    ("mathematics", 36500.0),      // 100 years — proofs are eternal
25    ("stock_prices", 1.0),         // 1 day
26    ("software_docs", 180.0),      // 6 months
27    ("historical_facts", 18250.0), // 50 years
28    ("security_advisories", 30.0), // 1 month
29    ("social_media", 7.0),         // 1 week
30    ("youtube_video", 180.0),      // 6 months
31    ("academic_papers", 1825.0),   // 5 years
32    ("news", 7.0),                 // 1 week
33    ("wikipedia", 365.0),          // 1 year
34];
35
36// ─── URL → domain category classifier ────────────────────────────────────────
37
38/// Return the domain-appropriate half-life (in days) for a URL.
39///
40/// Looks up the static `DEFAULT_HALF_LIVES` table directly — no heap allocation.
41/// Falls back to 180 days (6 months) for unrecognised domains.
42pub fn domain_half_life(url: &str) -> f64 {
43    let category = classify_url_domain(url);
44    DEFAULT_HALF_LIVES
45        .iter()
46        .find(|(cat, _)| *cat == category)
47        .map(|(_, hl)| *hl)
48        .unwrap_or(180.0)
49}
50
51/// Classify a URL into a content domain category based on hostname patterns.
52pub fn classify_url_domain(url: &str) -> &'static str {
53    let lower = url.to_lowercase();
54    if lower.contains("arxiv.org") || lower.contains("scholar.google") {
55        return "academic_papers";
56    }
57    if lower.contains("github.com") || lower.contains("docs.rs") || lower.contains("readthedocs") {
58        return "software_docs";
59    }
60    if lower.contains("cve.mitre") || lower.contains("nvd.nist") || lower.contains("security") {
61        return "security_advisories";
62    }
63    if lower.contains("nih.gov")
64        || lower.contains("pubmed")
65        || lower.contains("nejm")
66        || lower.contains("lancet")
67    {
68        return "medical_trials";
69    }
70    if lower.contains("law.")
71        || lower.contains("court")
72        || lower.contains("supremecourt")
73        || lower.contains("justice.gov")
74    {
75        return "legal_precedent";
76    }
77    if lower.contains("reddit.com")
78        || lower.contains("twitter.com")
79        || lower.contains("x.com")
80        || lower.contains("mastodon")
81        || lower.contains("bluesky")
82    {
83        return "social_media";
84    }
85    if lower.contains("wikipedia.org") {
86        return "wikipedia";
87    }
88    if lower.contains("youtube.com") || lower.contains("youtu.be") {
89        return "youtube_video";
90    }
91    if lower.contains("huggingface")
92        || lower.contains("paperswithcode")
93        || lower.contains("benchmarks")
94    {
95        return "ai_ml_benchmarks";
96    }
97    "news" // default
98}
99
100// ─── Types ───────────────────────────────────────────────────────────────────
101
102/// Staleness classification of a decayed confidence value.
103#[derive(Debug, Clone, PartialEq, serde::Serialize)]
104#[serde(rename_all = "snake_case")]
105pub enum Staleness {
106    Fresh,
107    PotentiallyStale,
108    Stale,
109}
110
111/// Result of applying EDF to a confidence score.
112#[derive(Debug, Clone, serde::Serialize)]
113pub struct DecayResult {
114    pub original_confidence: f64,
115    pub decayed_confidence: f64,
116    /// Content domain category used for half-life lookup.
117    pub domain_category: String,
118    pub age_days: f64,
119    pub half_life: f64,
120    pub staleness: Staleness,
121    pub flag: Option<String>,
122}
123
124// ─── Engine ──────────────────────────────────────────────────────────────────
125
126/// Evidence Decay Function engine.
127pub struct EvidenceDecayFunction {
128    half_lives: HashMap<String, f64>,
129    /// SQLite connection for self-calibration events.
130    calibration_db: Option<Connection>,
131}
132
133impl EvidenceDecayFunction {
134    /// Create a new EDF with default half-lives and no calibration DB.
135    pub fn new() -> Self {
136        let mut half_lives = HashMap::new();
137        for &(domain, hl) in DEFAULT_HALF_LIVES {
138            half_lives.insert(domain.to_string(), hl);
139        }
140        Self {
141            half_lives,
142            calibration_db: None,
143        }
144    }
145
146    /// Create with a calibration database that persists half-life adjustments.
147    pub fn with_calibration(db_path: &std::path::Path) -> Result<Self, FetchiumError> {
148        let conn = Connection::open(db_path)?;
149        enable_wal(&conn)?;
150        conn.execute_batch(
151            "CREATE TABLE IF NOT EXISTS calibration_events (
152                id                INTEGER PRIMARY KEY AUTOINCREMENT,
153                domain_category   TEXT    NOT NULL,
154                flagged_stale     INTEGER NOT NULL,
155                actually_valid    INTEGER NOT NULL,
156                recorded_at       TEXT    NOT NULL DEFAULT (datetime('now'))
157            );
158            CREATE TABLE IF NOT EXISTS half_life_overrides (
159                domain_category TEXT PRIMARY KEY,
160                half_life_days  REAL NOT NULL,
161                updated_at      TEXT NOT NULL DEFAULT (datetime('now'))
162            );",
163        )?;
164
165        let mut edf = Self::new();
166
167        // Load any persisted half-life overrides.
168        {
169            let mut stmt =
170                conn.prepare("SELECT domain_category, half_life_days FROM half_life_overrides")?;
171            let overrides: Vec<(String, f64)> = stmt
172                .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
173                .filter_map(|r| r.ok())
174                .collect();
175            for (cat, hl) in overrides {
176                edf.half_lives.insert(cat, hl);
177            }
178        }
179
180        edf.calibration_db = Some(conn);
181        Ok(edf)
182    }
183
184    /// Calculate decayed confidence for a claim of `age_days` old in `domain_category`.
185    ///
186    /// Clamps the result to [0.01, base_confidence] to prevent underflow or inflation.
187    pub fn decay(&self, base_confidence: f64, domain_category: &str, age_days: f64) -> DecayResult {
188        let half_life = self.get_half_life(domain_category);
189        let lambda = std::f64::consts::LN_2 / half_life;
190        let decayed = (base_confidence * (-lambda * age_days).exp())
191            .clamp(0.01_f64.min(base_confidence), base_confidence);
192
193        let staleness = if decayed < 0.3 {
194            Staleness::Stale
195        } else if decayed < 0.5 {
196            Staleness::PotentiallyStale
197        } else {
198            Staleness::Fresh
199        };
200
201        let flag = match &staleness {
202            Staleness::Stale => Some(format!(
203                "[stale: {age_days:.0} days old, half-life {half_life:.0} days]"
204            )),
205            Staleness::PotentiallyStale => Some(format!(
206                "[potentially stale: {age_days:.0} days old, half-life {half_life:.0} days]"
207            )),
208            Staleness::Fresh => None,
209        };
210
211        DecayResult {
212            original_confidence: base_confidence,
213            decayed_confidence: decayed,
214            domain_category: domain_category.to_string(),
215            age_days,
216            half_life,
217            staleness,
218            flag,
219        }
220    }
221
222    /// Self-calibration feedback: adjust half-life based on false-positive / false-negative flags.
223    ///
224    /// Requires at least 10 events per category before making adjustments.
225    pub fn calibrate(
226        &mut self,
227        domain_category: &str,
228        was_flagged_stale: bool,
229        actually_still_valid: bool,
230    ) -> Result<(), FetchiumError> {
231        if let Some(conn) = &self.calibration_db {
232            conn.execute(
233                "INSERT INTO calibration_events (domain_category, flagged_stale, actually_valid)
234                 VALUES (?1, ?2, ?3)",
235                rusqlite::params![
236                    domain_category,
237                    was_flagged_stale as i32,
238                    actually_still_valid as i32,
239                ],
240            )?;
241
242            // Count events for this category.
243            let event_count: i64 = conn.query_row(
244                "SELECT COUNT(*) FROM calibration_events WHERE domain_category = ?1",
245                [domain_category],
246                |row| row.get(0),
247            )?;
248
249            if event_count >= 10 {
250                let current = self.get_half_life(domain_category);
251                let new_half_life = if was_flagged_stale && actually_still_valid {
252                    // False positive: content was still valid → increase half-life by 10%
253                    current * 1.1
254                } else if !was_flagged_stale && !actually_still_valid {
255                    // False negative: content was stale but we didn't flag it → decrease by 10%
256                    (current * 0.9).max(1.0)
257                } else {
258                    current
259                };
260
261                if (new_half_life - current).abs() > 0.01 {
262                    self.half_lives
263                        .insert(domain_category.to_string(), new_half_life);
264                    conn.execute(
265                        "INSERT INTO half_life_overrides (domain_category, half_life_days)
266                         VALUES (?1, ?2)
267                         ON CONFLICT(domain_category) DO UPDATE SET
268                            half_life_days = ?2,
269                            updated_at = datetime('now')",
270                        rusqlite::params![domain_category, new_half_life],
271                    )?;
272                    tracing::info!(
273                        domain = domain_category,
274                        old_hl = current,
275                        new_hl = new_half_life,
276                        "EDF: half-life adjusted"
277                    );
278                }
279            }
280        }
281        Ok(())
282    }
283
284    fn get_half_life(&self, domain_category: &str) -> f64 {
285        self.half_lives
286            .get(domain_category)
287            .copied()
288            .unwrap_or(180.0) // default: 6 months
289    }
290}
291
292impl Default for EvidenceDecayFunction {
293    fn default() -> Self {
294        Self::new()
295    }
296}
297
298#[cfg(test)]
299mod tests {
300    use super::*;
301
302    #[test]
303    fn ai_benchmarks_halve_after_90_days() {
304        let edf = EvidenceDecayFunction::new();
305        let result = edf.decay(0.9, "ai_ml_benchmarks", 90.0);
306        // After one half-life, confidence should be ~0.45
307        assert!(
308            (result.decayed_confidence - 0.45).abs() < 0.02,
309            "decayed={:.3}",
310            result.decayed_confidence
311        );
312    }
313
314    #[test]
315    fn math_barely_decays_in_a_year() {
316        let edf = EvidenceDecayFunction::new();
317        let result = edf.decay(0.9, "mathematics", 365.0);
318        assert!(
319            result.decayed_confidence > 0.88,
320            "decayed={:.3}",
321            result.decayed_confidence
322        );
323        assert_eq!(result.staleness, Staleness::Fresh);
324    }
325
326    #[test]
327    fn stale_content_gets_flag() {
328        let edf = EvidenceDecayFunction::new();
329        let result = edf.decay(0.9, "social_media", 60.0); // 60 days, 7-day half-life
330        assert!(result.staleness == Staleness::Stale);
331        assert!(result.flag.is_some());
332    }
333
334    #[test]
335    fn classify_url_domain_arxiv() {
336        assert_eq!(
337            classify_url_domain("https://arxiv.org/abs/2301.12345"),
338            "academic_papers"
339        );
340    }
341
342    #[test]
343    fn classify_url_domain_github() {
344        assert_eq!(
345            classify_url_domain("https://github.com/rust-lang/rust"),
346            "software_docs"
347        );
348    }
349}