Skip to main content

fetchium_core/intelligence/
mod.rs

1//! Intelligence algorithms — PIE, ToTR, CRP, EDF, SGT, CCE, ACS (PRD §31-39).
2//!
3//! This module houses 7 novel intelligence algorithms that give Fetchium
4//! persistent learning across sessions and adversarial robustness.
5
6pub mod acs;
7pub mod cce;
8pub mod crp;
9pub mod edf;
10pub mod pie;
11pub mod sgt;
12pub mod totr;
13
14// ─── Shared types ────────────────────────────────────────────────────────────
15
16/// An observation produced by a search, fetch, or extraction operation.
17#[derive(Debug, Clone)]
18pub enum Observation {
19    /// A search completed — record query + result domains.
20    SearchCompleted {
21        query: String,
22        /// URLs returned by the search
23        result_urls: Vec<String>,
24    },
25    /// A single fetch attempt — success or failure.
26    FetchAttempt {
27        domain: String,
28        url: String,
29        /// CEP layer attempted (1-5)
30        extraction_layer: u8,
31        success: bool,
32        error: Option<String>,
33        duration_ms: u64,
34    },
35    /// Relevance feedback for a domain after content extraction.
36    ContentRelevance {
37        domain: String,
38        query: String,
39        relevance_score: f64,
40    },
41    /// A named entity discovered during research.
42    EntityDiscovered {
43        name: String,
44        entity_type: String,
45        source_url: String,
46    },
47    /// A relationship between two entities.
48    RelationshipDiscovered {
49        entity_a: String,
50        entity_b: String,
51        relation: String,
52        source_url: String,
53    },
54}
55
56/// Statistics for a single intelligence layer.
57#[derive(Debug, Clone, Default, serde::Serialize)]
58pub struct LayerStats {
59    pub name: String,
60    pub record_count: u64,
61    pub db_size_bytes: u64,
62}
63
64// ─── Helpers ─────────────────────────────────────────────────────────────────
65
66/// Returns the default intelligence data directory: `~/.fetchium/intelligence/`.
67pub fn intelligence_data_dir() -> std::path::PathBuf {
68    dirs::home_dir()
69        .unwrap_or_else(|| std::path::PathBuf::from("."))
70        .join(".fetchium")
71        .join("intelligence")
72}
73
74/// Enable WAL mode on a rusqlite connection for concurrent read/write safety.
75pub(crate) fn enable_wal(conn: &rusqlite::Connection) -> rusqlite::Result<()> {
76    conn.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;")?;
77    Ok(())
78}
79
80/// SHA-256 hex digest of a UTF-8 string.
81pub(crate) fn sha256_hex(input: &str) -> String {
82    use sha2::Digest;
83    let mut h = sha2::Sha256::new();
84    h.update(input.as_bytes());
85    h.finalize().iter().map(|b| format!("{b:02x}")).collect()
86}
87
88/// Simple normalised string similarity (Jaccard on character bigrams).
89/// Returns 0.0 (totally different) to 1.0 (identical).
90pub(crate) fn string_similarity(a: &str, b: &str) -> f64 {
91    let bigrams = |s: &str| -> std::collections::HashSet<[char; 2]> {
92        let chars: Vec<char> = s.to_lowercase().chars().collect();
93        chars.windows(2).map(|w| [w[0], w[1]]).collect()
94    };
95
96    let a_set = bigrams(a);
97    let b_set = bigrams(b);
98
99    if a_set.is_empty() && b_set.is_empty() {
100        return 1.0;
101    }
102    if a_set.is_empty() || b_set.is_empty() {
103        return 0.0;
104    }
105
106    let intersection = a_set.intersection(&b_set).count();
107    let union = a_set.union(&b_set).count();
108    intersection as f64 / union as f64
109}