use crate::error::FetchiumError;
use crate::index::document::{IndexStats, IndexedDocument};
use chrono::Utc;
use rusqlite::{Connection, OptionalExtension};
use sha2::{Digest, Sha256};
use std::path::Path;
use std::sync::Mutex;
use tracing::debug;
pub struct DocumentStore {
conn: Mutex<Connection>,
}
impl DocumentStore {
pub fn new(db_path: &Path) -> Result<Self, FetchiumError> {
if let Some(p) = db_path.parent() {
std::fs::create_dir_all(p)?;
}
let conn = Connection::open(db_path)?;
conn.execute_batch(
"PRAGMA journal_mode=WAL;
CREATE TABLE IF NOT EXISTS documents (
id INTEGER PRIMARY KEY AUTOINCREMENT,
url TEXT NOT NULL UNIQUE,
title TEXT NOT NULL DEFAULT '',
content TEXT NOT NULL DEFAULT '',
domain TEXT NOT NULL DEFAULT '',
fetched_at TEXT NOT NULL,
content_hash TEXT NOT NULL,
has_embedding INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_docs_domain
ON documents(domain);
CREATE INDEX IF NOT EXISTS idx_docs_hash
ON documents(content_hash);",
)?;
Ok(Self {
conn: Mutex::new(conn),
})
}
pub fn upsert(&self, doc: &IndexedDocument) -> Result<(u64, bool), FetchiumError> {
let hash = &doc.content_hash;
let conn = self.conn.lock().unwrap();
let existing_hash: Option<String> = conn
.query_row(
"SELECT content_hash FROM documents WHERE url = ?1",
[&doc.url],
|r| r.get(0),
)
.optional()?;
let changed = existing_hash.as_deref() != Some(hash.as_str());
conn.execute(
"INSERT INTO documents (url, title, content, domain, fetched_at, content_hash, has_embedding)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
ON CONFLICT(url) DO UPDATE SET
title = excluded.title,
content = excluded.content,
domain = excluded.domain,
fetched_at = excluded.fetched_at,
content_hash = excluded.content_hash,
has_embedding = CASE
WHEN excluded.content_hash != documents.content_hash THEN 0
ELSE documents.has_embedding
END",
rusqlite::params![
doc.url,
doc.title,
doc.content,
doc.domain,
doc.fetched_at.to_rfc3339(),
hash,
doc.embedding.is_some() as i64,
],
)?;
let id: i64 =
conn.query_row("SELECT id FROM documents WHERE url = ?1", [&doc.url], |r| {
r.get(0)
})?;
debug!(
"Document upserted: id={id}, url={}, changed={changed}",
doc.url
);
Ok((id as u64, changed))
}
pub fn get_by_id(&self, id: u64) -> Result<Option<IndexedDocument>, FetchiumError> {
let conn = self.conn.lock().unwrap();
let result = conn
.query_row(
"SELECT id, url, title, content, domain, fetched_at, content_hash
FROM documents WHERE id = ?1",
[id],
|r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
r.get::<_, String>(3)?,
r.get::<_, String>(4)?,
r.get::<_, String>(5)?,
r.get::<_, String>(6)?,
))
},
)
.optional()?;
Ok(result.map(
|(id, url, title, content, domain, fetched_at, content_hash)| IndexedDocument {
id: id as u64,
url,
title,
content,
domain,
fetched_at: fetched_at.parse().unwrap_or_else(|_| Utc::now()),
content_hash,
embedding: None,
},
))
}
pub fn get_by_ids(&self, ids: &[u64]) -> Result<Vec<IndexedDocument>, FetchiumError> {
if ids.is_empty() {
return Ok(Vec::new());
}
let mut docs = Vec::with_capacity(ids.len());
for &id in ids {
if let Some(doc) = self.get_by_id(id)? {
docs.push(doc);
}
}
Ok(docs)
}
pub fn documents_without_embeddings(&self) -> Result<Vec<IndexedDocument>, FetchiumError> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT id, url, title, content, domain, fetched_at, content_hash
FROM documents WHERE has_embedding = 0",
)?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
r.get::<_, String>(3)?,
r.get::<_, String>(4)?,
r.get::<_, String>(5)?,
r.get::<_, String>(6)?,
))
})?;
let mut docs = Vec::new();
for row in rows {
let (id, url, title, content, domain, fetched_at, content_hash) = row?;
docs.push(IndexedDocument {
id: id as u64,
url,
title,
content,
domain,
fetched_at: fetched_at.parse().unwrap_or_else(|_| Utc::now()),
content_hash,
embedding: None,
});
}
Ok(docs)
}
pub fn mark_embedded(&self, id: u64) -> Result<(), FetchiumError> {
let conn = self.conn.lock().unwrap();
conn.execute("UPDATE documents SET has_embedding = 1 WHERE id = ?1", [id])?;
Ok(())
}
pub fn stats(&self) -> Result<IndexStats, FetchiumError> {
let conn = self.conn.lock().unwrap();
let doc_count: i64 = conn.query_row("SELECT COUNT(*) FROM documents", [], |r| r.get(0))?;
let embedded_count: i64 = conn.query_row(
"SELECT COUNT(*) FROM documents WHERE has_embedding = 1",
[],
|r| r.get(0),
)?;
Ok(IndexStats {
document_count: doc_count as usize,
embedded_count: embedded_count as usize,
index_size_bytes: 0, vector_index_ready: false, })
}
pub fn search(&self, query: &str, limit: usize) -> Result<Vec<IndexedDocument>, FetchiumError> {
let conn = self.conn.lock().unwrap();
let pattern = format!("%{}%", query.replace('%', r"\%").replace('_', r"\_"));
let mut stmt = conn.prepare(
"SELECT id, url, title, content, domain, fetched_at, content_hash
FROM documents
WHERE title LIKE ?1 ESCAPE '\\' OR content LIKE ?1 ESCAPE '\\'
ORDER BY id DESC
LIMIT ?2",
)?;
let rows = stmt.query_map(rusqlite::params![pattern, limit as i64], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
r.get::<_, String>(3)?,
r.get::<_, String>(4)?,
r.get::<_, String>(5)?,
r.get::<_, String>(6)?,
))
})?;
let mut docs = Vec::new();
for row in rows {
let (id, url, title, content, domain, fetched_at, content_hash) = row?;
docs.push(IndexedDocument {
id: id as u64,
url,
title,
content,
domain,
fetched_at: fetched_at.parse().unwrap_or_else(|_| Utc::now()),
content_hash,
embedding: None,
});
}
Ok(docs)
}
pub fn clear(&self) -> Result<usize, FetchiumError> {
let conn = self.conn.lock().unwrap();
let n = conn.execute("DELETE FROM documents", [])?;
debug!("DocumentStore cleared: {n} documents removed");
Ok(n)
}
pub fn content_hash(content: &str) -> String {
format!("{:x}", Sha256::digest(content.as_bytes()))
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::NamedTempFile;
fn make_store() -> (DocumentStore, NamedTempFile) {
let tmp = NamedTempFile::new().unwrap();
let store = DocumentStore::new(tmp.path()).unwrap();
(store, tmp)
}
fn make_doc(n: u64) -> IndexedDocument {
IndexedDocument {
id: n,
url: format!("https://example.com/{n}"),
title: format!("Doc {n}"),
content: format!("Content for document {n}"),
domain: "example.com".into(),
fetched_at: Utc::now(),
content_hash: format!("hash{n}"),
embedding: None,
}
}
#[test]
fn upsert_and_get() {
let (store, _tmp) = make_store();
let doc = make_doc(1);
let (id, new) = store.upsert(&doc).unwrap();
assert!(new);
let loaded = store.get_by_id(id).unwrap().unwrap();
assert_eq!(loaded.url, "https://example.com/1");
}
#[test]
fn upsert_same_hash_not_changed() {
let (store, _tmp) = make_store();
let doc = make_doc(2);
store.upsert(&doc).unwrap();
let (_, changed) = store.upsert(&doc).unwrap();
assert!(!changed);
}
#[test]
fn stats_count() {
let (store, _tmp) = make_store();
store.upsert(&make_doc(1)).unwrap();
store.upsert(&make_doc(2)).unwrap();
let stats = store.stats().unwrap();
assert_eq!(stats.document_count, 2);
assert_eq!(stats.embedded_count, 0);
}
#[test]
fn mark_embedded_updates_flag() {
let (store, _tmp) = make_store();
let (id, _) = store.upsert(&make_doc(3)).unwrap();
store.mark_embedded(id).unwrap();
let stats = store.stats().unwrap();
assert_eq!(stats.embedded_count, 1);
}
#[test]
fn content_hash_deterministic() {
let h1 = DocumentStore::content_hash("hello");
let h2 = DocumentStore::content_hash("hello");
assert_eq!(h1, h2);
assert_ne!(h1, DocumentStore::content_hash("world"));
}
}