pub mod acs;
pub mod cce;
pub mod crp;
pub mod edf;
pub mod pie;
pub mod sgt;
pub mod totr;
#[derive(Debug, Clone)]
pub enum Observation {
SearchCompleted {
query: String,
result_urls: Vec<String>,
},
FetchAttempt {
domain: String,
url: String,
extraction_layer: u8,
success: bool,
error: Option<String>,
duration_ms: u64,
},
ContentRelevance {
domain: String,
query: String,
relevance_score: f64,
},
EntityDiscovered {
name: String,
entity_type: String,
source_url: String,
},
RelationshipDiscovered {
entity_a: String,
entity_b: String,
relation: String,
source_url: String,
},
}
#[derive(Debug, Clone, Default, serde::Serialize)]
pub struct LayerStats {
pub name: String,
pub record_count: u64,
pub db_size_bytes: u64,
}
pub fn intelligence_data_dir() -> std::path::PathBuf {
dirs::home_dir()
.unwrap_or_else(|| std::path::PathBuf::from("."))
.join(".fetchium")
.join("intelligence")
}
pub(crate) fn enable_wal(conn: &rusqlite::Connection) -> rusqlite::Result<()> {
conn.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;")?;
Ok(())
}
pub(crate) fn sha256_hex(input: &str) -> String {
use sha2::Digest;
let mut h = sha2::Sha256::new();
h.update(input.as_bytes());
h.finalize().iter().map(|b| format!("{b:02x}")).collect()
}
pub(crate) fn string_similarity(a: &str, b: &str) -> f64 {
let bigrams = |s: &str| -> std::collections::HashSet<[char; 2]> {
let chars: Vec<char> = s.to_lowercase().chars().collect();
chars.windows(2).map(|w| [w[0], w[1]]).collect()
};
let a_set = bigrams(a);
let b_set = bigrams(b);
if a_set.is_empty() && b_set.is_empty() {
return 1.0;
}
if a_set.is_empty() || b_set.is_empty() {
return 0.0;
}
let intersection = a_set.intersection(&b_set).count();
let union = a_set.union(&b_set).count();
intersection as f64 / union as f64
}