Skip to main content

fetchium_core/intelligence/pie/
fpm.rs

1//! Failure Pattern Memory (FPM) — tracks extraction success/failure per domain+layer.
2//!
3//! Schema:
4//! - `extraction_successes(domain, layer, count, total_time_ms)`
5//! - `extraction_failures(domain, layer, error_type, count, last_occurred)`
6
7use rusqlite::Connection;
8use std::sync::Mutex;
9
10use crate::error::FetchiumError;
11use crate::intelligence::enable_wal;
12
13pub struct FailurePatternMemory {
14    conn: Mutex<Connection>,
15}
16
17impl FailurePatternMemory {
18    pub fn new(db_path: &std::path::Path) -> Result<Self, FetchiumError> {
19        let conn = Connection::open(db_path)?;
20        enable_wal(&conn)?;
21        conn.execute_batch(
22            "CREATE TABLE IF NOT EXISTS extraction_successes (
23                domain       TEXT    NOT NULL,
24                layer        INTEGER NOT NULL,
25                count        INTEGER NOT NULL DEFAULT 0,
26                total_time_ms INTEGER NOT NULL DEFAULT 0,
27                PRIMARY KEY (domain, layer)
28            );
29            CREATE TABLE IF NOT EXISTS extraction_failures (
30                domain       TEXT    NOT NULL,
31                layer        INTEGER NOT NULL,
32                error_type   TEXT    NOT NULL,
33                count        INTEGER NOT NULL DEFAULT 0,
34                last_occurred TEXT   NOT NULL DEFAULT (datetime('now')),
35                PRIMARY KEY (domain, layer, error_type)
36            );",
37        )?;
38        Ok(Self {
39            conn: Mutex::new(conn),
40        })
41    }
42
43    /// Record an extraction attempt (success or failure) for `domain` at CEP `layer`.
44    pub fn record_attempt(
45        &self,
46        domain: &str,
47        layer: u8,
48        success: bool,
49        error: Option<&str>,
50        duration_ms: u64,
51    ) -> Result<(), FetchiumError> {
52        let conn = self.conn.lock().unwrap();
53        if success {
54            conn.execute(
55                "INSERT INTO extraction_successes (domain, layer, count, total_time_ms)
56                 VALUES (?1, ?2, 1, ?3)
57                 ON CONFLICT(domain, layer) DO UPDATE SET
58                    count         = count + 1,
59                    total_time_ms = total_time_ms + ?3",
60                rusqlite::params![domain, layer as i64, duration_ms as i64],
61            )?;
62        } else {
63            conn.execute(
64                "INSERT INTO extraction_failures (domain, layer, error_type, count, last_occurred)
65                 VALUES (?1, ?2, ?3, 1, datetime('now'))
66                 ON CONFLICT(domain, layer, error_type) DO UPDATE SET
67                    count         = count + 1,
68                    last_occurred = datetime('now')",
69                rusqlite::params![domain, layer as i64, error.unwrap_or("unknown")],
70            )?;
71        }
72        Ok(())
73    }
74
75    /// Recommend the best CEP layer for `domain` based on historical data.
76    ///
77    /// Returns `(recommended_layer, confidence)`.
78    /// Default if no history: layer 2, confidence 0.5.
79    pub fn recommend_layer(&self, domain: &str) -> Result<(u8, f64), FetchiumError> {
80        let conn = self.conn.lock().unwrap();
81
82        let mut stmt = conn.prepare(
83            "SELECT s.layer, s.count,
84                    COALESCE((SELECT SUM(f.count)
85                              FROM extraction_failures f
86                              WHERE f.domain = s.domain AND f.layer = s.layer), 0)
87             FROM extraction_successes s
88             WHERE s.domain = ?1
89             ORDER BY s.count DESC",
90        )?;
91
92        let results: Vec<(u8, i64, i64)> = stmt
93            .query_map([domain], |row| {
94                Ok((
95                    row.get::<_, i64>(0)? as u8,
96                    row.get::<_, i64>(1)?,
97                    row.get::<_, i64>(2)?,
98                ))
99            })?
100            .filter_map(|r| r.ok())
101            .collect();
102
103        if let Some((layer, successes, failures)) = results.first() {
104            let total = (*successes + *failures).max(1) as f64;
105            let confidence = *successes as f64 / total;
106            Ok((*layer, confidence))
107        } else {
108            Ok((2, 0.5)) // default: Layer 2 (HTTP + readability)
109        }
110    }
111
112    /// Total number of distinct failure patterns recorded.
113    pub fn pattern_count(&self) -> Result<u64, FetchiumError> {
114        let conn = self.conn.lock().unwrap();
115        let n: i64 = conn.query_row("SELECT COUNT(*) FROM extraction_failures", [], |row| {
116            row.get(0)
117        })?;
118        Ok(n as u64)
119    }
120
121    /// Reset all failure/success patterns.
122    pub fn reset(&self) -> Result<(), FetchiumError> {
123        let conn = self.conn.lock().unwrap();
124        conn.execute_batch(
125            "DELETE FROM extraction_successes;
126             DELETE FROM extraction_failures;
127             VACUUM;",
128        )?;
129        Ok(())
130    }
131}
132
133#[cfg(test)]
134mod tests {
135    use super::*;
136    use tempfile::NamedTempFile;
137
138    #[test]
139    fn recommend_layer_returns_best_success_layer() {
140        let tmp = NamedTempFile::new().unwrap();
141        let fpm = FailurePatternMemory::new(tmp.path()).unwrap();
142        // Layer 3 succeeds 5× for "spa.com"
143        for _ in 0..5 {
144            fpm.record_attempt("spa.com", 1, false, Some("timeout"), 0)
145                .unwrap();
146            fpm.record_attempt("spa.com", 3, true, None, 500).unwrap();
147        }
148        let (layer, conf) = fpm.recommend_layer("spa.com").unwrap();
149        assert_eq!(layer, 3);
150        assert!(conf > 0.5, "conf={conf:.2}");
151    }
152
153    #[test]
154    fn no_history_returns_default() {
155        let tmp = NamedTempFile::new().unwrap();
156        let fpm = FailurePatternMemory::new(tmp.path()).unwrap();
157        let (layer, conf) = fpm.recommend_layer("new-site.io").unwrap();
158        assert_eq!(layer, 2);
159        assert!((conf - 0.5).abs() < 1e-9);
160    }
161}