fetchium_core/intelligence/
edf.rs1use std::collections::HashMap;
10
11use rusqlite::Connection;
12
13use crate::error::FetchiumError;
14use crate::intelligence::enable_wal;
15
16static DEFAULT_HALF_LIVES: &[(&str, f64)] = &[
20 ("ai_ml_benchmarks", 90.0), ("tech_news", 14.0), ("medical_trials", 730.0), ("legal_precedent", 3650.0), ("mathematics", 36500.0), ("stock_prices", 1.0), ("software_docs", 180.0), ("historical_facts", 18250.0), ("security_advisories", 30.0), ("social_media", 7.0), ("youtube_video", 180.0), ("academic_papers", 1825.0), ("news", 7.0), ("wikipedia", 365.0), ];
35
36pub fn domain_half_life(url: &str) -> f64 {
43 let category = classify_url_domain(url);
44 DEFAULT_HALF_LIVES
45 .iter()
46 .find(|(cat, _)| *cat == category)
47 .map(|(_, hl)| *hl)
48 .unwrap_or(180.0)
49}
50
51pub fn classify_url_domain(url: &str) -> &'static str {
53 let lower = url.to_lowercase();
54 if lower.contains("arxiv.org") || lower.contains("scholar.google") {
55 return "academic_papers";
56 }
57 if lower.contains("github.com") || lower.contains("docs.rs") || lower.contains("readthedocs") {
58 return "software_docs";
59 }
60 if lower.contains("cve.mitre") || lower.contains("nvd.nist") || lower.contains("security") {
61 return "security_advisories";
62 }
63 if lower.contains("nih.gov")
64 || lower.contains("pubmed")
65 || lower.contains("nejm")
66 || lower.contains("lancet")
67 {
68 return "medical_trials";
69 }
70 if lower.contains("law.")
71 || lower.contains("court")
72 || lower.contains("supremecourt")
73 || lower.contains("justice.gov")
74 {
75 return "legal_precedent";
76 }
77 if lower.contains("reddit.com")
78 || lower.contains("twitter.com")
79 || lower.contains("x.com")
80 || lower.contains("mastodon")
81 || lower.contains("bluesky")
82 {
83 return "social_media";
84 }
85 if lower.contains("wikipedia.org") {
86 return "wikipedia";
87 }
88 if lower.contains("youtube.com") || lower.contains("youtu.be") {
89 return "youtube_video";
90 }
91 if lower.contains("huggingface")
92 || lower.contains("paperswithcode")
93 || lower.contains("benchmarks")
94 {
95 return "ai_ml_benchmarks";
96 }
97 "news" }
99
100#[derive(Debug, Clone, PartialEq, serde::Serialize)]
104#[serde(rename_all = "snake_case")]
105pub enum Staleness {
106 Fresh,
107 PotentiallyStale,
108 Stale,
109}
110
111#[derive(Debug, Clone, serde::Serialize)]
113pub struct DecayResult {
114 pub original_confidence: f64,
115 pub decayed_confidence: f64,
116 pub domain_category: String,
118 pub age_days: f64,
119 pub half_life: f64,
120 pub staleness: Staleness,
121 pub flag: Option<String>,
122}
123
124pub struct EvidenceDecayFunction {
128 half_lives: HashMap<String, f64>,
129 calibration_db: Option<Connection>,
131}
132
133impl EvidenceDecayFunction {
134 pub fn new() -> Self {
136 let mut half_lives = HashMap::new();
137 for &(domain, hl) in DEFAULT_HALF_LIVES {
138 half_lives.insert(domain.to_string(), hl);
139 }
140 Self {
141 half_lives,
142 calibration_db: None,
143 }
144 }
145
146 pub fn with_calibration(db_path: &std::path::Path) -> Result<Self, FetchiumError> {
148 let conn = Connection::open(db_path)?;
149 enable_wal(&conn)?;
150 conn.execute_batch(
151 "CREATE TABLE IF NOT EXISTS calibration_events (
152 id INTEGER PRIMARY KEY AUTOINCREMENT,
153 domain_category TEXT NOT NULL,
154 flagged_stale INTEGER NOT NULL,
155 actually_valid INTEGER NOT NULL,
156 recorded_at TEXT NOT NULL DEFAULT (datetime('now'))
157 );
158 CREATE TABLE IF NOT EXISTS half_life_overrides (
159 domain_category TEXT PRIMARY KEY,
160 half_life_days REAL NOT NULL,
161 updated_at TEXT NOT NULL DEFAULT (datetime('now'))
162 );",
163 )?;
164
165 let mut edf = Self::new();
166
167 {
169 let mut stmt =
170 conn.prepare("SELECT domain_category, half_life_days FROM half_life_overrides")?;
171 let overrides: Vec<(String, f64)> = stmt
172 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
173 .filter_map(|r| r.ok())
174 .collect();
175 for (cat, hl) in overrides {
176 edf.half_lives.insert(cat, hl);
177 }
178 }
179
180 edf.calibration_db = Some(conn);
181 Ok(edf)
182 }
183
184 pub fn decay(&self, base_confidence: f64, domain_category: &str, age_days: f64) -> DecayResult {
188 let half_life = self.get_half_life(domain_category);
189 let lambda = std::f64::consts::LN_2 / half_life;
190 let decayed = (base_confidence * (-lambda * age_days).exp())
191 .clamp(0.01_f64.min(base_confidence), base_confidence);
192
193 let staleness = if decayed < 0.3 {
194 Staleness::Stale
195 } else if decayed < 0.5 {
196 Staleness::PotentiallyStale
197 } else {
198 Staleness::Fresh
199 };
200
201 let flag = match &staleness {
202 Staleness::Stale => Some(format!(
203 "[stale: {age_days:.0} days old, half-life {half_life:.0} days]"
204 )),
205 Staleness::PotentiallyStale => Some(format!(
206 "[potentially stale: {age_days:.0} days old, half-life {half_life:.0} days]"
207 )),
208 Staleness::Fresh => None,
209 };
210
211 DecayResult {
212 original_confidence: base_confidence,
213 decayed_confidence: decayed,
214 domain_category: domain_category.to_string(),
215 age_days,
216 half_life,
217 staleness,
218 flag,
219 }
220 }
221
222 pub fn calibrate(
226 &mut self,
227 domain_category: &str,
228 was_flagged_stale: bool,
229 actually_still_valid: bool,
230 ) -> Result<(), FetchiumError> {
231 if let Some(conn) = &self.calibration_db {
232 conn.execute(
233 "INSERT INTO calibration_events (domain_category, flagged_stale, actually_valid)
234 VALUES (?1, ?2, ?3)",
235 rusqlite::params![
236 domain_category,
237 was_flagged_stale as i32,
238 actually_still_valid as i32,
239 ],
240 )?;
241
242 let event_count: i64 = conn.query_row(
244 "SELECT COUNT(*) FROM calibration_events WHERE domain_category = ?1",
245 [domain_category],
246 |row| row.get(0),
247 )?;
248
249 if event_count >= 10 {
250 let current = self.get_half_life(domain_category);
251 let new_half_life = if was_flagged_stale && actually_still_valid {
252 current * 1.1
254 } else if !was_flagged_stale && !actually_still_valid {
255 (current * 0.9).max(1.0)
257 } else {
258 current
259 };
260
261 if (new_half_life - current).abs() > 0.01 {
262 self.half_lives
263 .insert(domain_category.to_string(), new_half_life);
264 conn.execute(
265 "INSERT INTO half_life_overrides (domain_category, half_life_days)
266 VALUES (?1, ?2)
267 ON CONFLICT(domain_category) DO UPDATE SET
268 half_life_days = ?2,
269 updated_at = datetime('now')",
270 rusqlite::params![domain_category, new_half_life],
271 )?;
272 tracing::info!(
273 domain = domain_category,
274 old_hl = current,
275 new_hl = new_half_life,
276 "EDF: half-life adjusted"
277 );
278 }
279 }
280 }
281 Ok(())
282 }
283
284 fn get_half_life(&self, domain_category: &str) -> f64 {
285 self.half_lives
286 .get(domain_category)
287 .copied()
288 .unwrap_or(180.0) }
290}
291
292impl Default for EvidenceDecayFunction {
293 fn default() -> Self {
294 Self::new()
295 }
296}
297
298#[cfg(test)]
299mod tests {
300 use super::*;
301
302 #[test]
303 fn ai_benchmarks_halve_after_90_days() {
304 let edf = EvidenceDecayFunction::new();
305 let result = edf.decay(0.9, "ai_ml_benchmarks", 90.0);
306 assert!(
308 (result.decayed_confidence - 0.45).abs() < 0.02,
309 "decayed={:.3}",
310 result.decayed_confidence
311 );
312 }
313
314 #[test]
315 fn math_barely_decays_in_a_year() {
316 let edf = EvidenceDecayFunction::new();
317 let result = edf.decay(0.9, "mathematics", 365.0);
318 assert!(
319 result.decayed_confidence > 0.88,
320 "decayed={:.3}",
321 result.decayed_confidence
322 );
323 assert_eq!(result.staleness, Staleness::Fresh);
324 }
325
326 #[test]
327 fn stale_content_gets_flag() {
328 let edf = EvidenceDecayFunction::new();
329 let result = edf.decay(0.9, "social_media", 60.0); assert!(result.staleness == Staleness::Stale);
331 assert!(result.flag.is_some());
332 }
333
334 #[test]
335 fn classify_url_domain_arxiv() {
336 assert_eq!(
337 classify_url_domain("https://arxiv.org/abs/2301.12345"),
338 "academic_papers"
339 );
340 }
341
342 #[test]
343 fn classify_url_domain_github() {
344 assert_eq!(
345 classify_url_domain("https://github.com/rust-lang/rust"),
346 "software_docs"
347 );
348 }
349}