fetchium_core/intelligence/
mod.rs1pub mod acs;
7pub mod cce;
8pub mod crp;
9pub mod edf;
10pub mod pie;
11pub mod sgt;
12pub mod totr;
13
14#[derive(Debug, Clone)]
18pub enum Observation {
19 SearchCompleted {
21 query: String,
22 result_urls: Vec<String>,
24 },
25 FetchAttempt {
27 domain: String,
28 url: String,
29 extraction_layer: u8,
31 success: bool,
32 error: Option<String>,
33 duration_ms: u64,
34 },
35 ContentRelevance {
37 domain: String,
38 query: String,
39 relevance_score: f64,
40 },
41 EntityDiscovered {
43 name: String,
44 entity_type: String,
45 source_url: String,
46 },
47 RelationshipDiscovered {
49 entity_a: String,
50 entity_b: String,
51 relation: String,
52 source_url: String,
53 },
54}
55
56#[derive(Debug, Clone, Default, serde::Serialize)]
58pub struct LayerStats {
59 pub name: String,
60 pub record_count: u64,
61 pub db_size_bytes: u64,
62}
63
64pub fn intelligence_data_dir() -> std::path::PathBuf {
68 dirs::home_dir()
69 .unwrap_or_else(|| std::path::PathBuf::from("."))
70 .join(".fetchium")
71 .join("intelligence")
72}
73
74pub(crate) fn enable_wal(conn: &rusqlite::Connection) -> rusqlite::Result<()> {
76 conn.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;")?;
77 Ok(())
78}
79
80pub(crate) fn sha256_hex(input: &str) -> String {
82 use sha2::Digest;
83 let mut h = sha2::Sha256::new();
84 h.update(input.as_bytes());
85 h.finalize().iter().map(|b| format!("{b:02x}")).collect()
86}
87
88pub(crate) fn string_similarity(a: &str, b: &str) -> f64 {
91 let bigrams = |s: &str| -> std::collections::HashSet<[char; 2]> {
92 let chars: Vec<char> = s.to_lowercase().chars().collect();
93 chars.windows(2).map(|w| [w[0], w[1]]).collect()
94 };
95
96 let a_set = bigrams(a);
97 let b_set = bigrams(b);
98
99 if a_set.is_empty() && b_set.is_empty() {
100 return 1.0;
101 }
102 if a_set.is_empty() || b_set.is_empty() {
103 return 0.0;
104 }
105
106 let intersection = a_set.intersection(&b_set).count();
107 let union = a_set.union(&b_set).count();
108 intersection as f64 / union as f64
109}