use anyhow::Result;
use rusqlite::{OptionalExtension, params};
use uuid::Uuid;
use super::{SqliteStore, blob_to_embedding, cosine_blob, embedding_to_blob, parse_uuid};
use crate::chunk;
use crate::schema::MemoryType;
#[derive(Debug, Clone, PartialEq)]
pub struct ChunkWork {
pub memory_id: Uuid,
pub content_hash: String,
pub chunks: Vec<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct MemoryChunk {
pub ord: usize,
pub text: String,
pub embedding: Vec<f32>,
pub content_hash: String,
}
impl SqliteStore {
pub fn chunks_needing_embedding(&self, limit: usize) -> Result<Vec<ChunkWork>> {
Ok(self.scan_chunk_work(limit)?.0)
}
pub fn scan_chunk_work(&self, limit: usize) -> Result<(Vec<ChunkWork>, usize)> {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT m.id, m.content, \
(SELECT c.content_hash FROM memory_chunks c WHERE c.memory_id = m.id LIMIT 1) \
FROM memories m \
WHERE m.tier = 'hot' AND m.superseded_by IS NULL AND length(m.content) >= ?1 \
ORDER BY m.rowid",
)?;
let mut rows = stmt.query(params![(2 * chunk::MIN_CHUNK_CHARS) as i64])?;
let mut out = Vec::new();
let mut pending = 0;
while let Some(row) = rows.next()? {
let content: String = row.get(1)?;
let stored: Option<String> = row.get(2)?;
let content_hash = chunk::content_hash(&content);
if stored.as_deref() == Some(content_hash.as_str()) {
continue;
}
let chunks = chunk::split(&content);
if chunks.is_empty() {
continue;
}
pending += 1;
if out.len() < limit {
out.push(ChunkWork {
memory_id: parse_uuid(&row.get::<_, String>(0)?)?,
content_hash,
chunks,
});
}
}
Ok((out, pending))
}
pub fn chunk_progress(&self) -> Result<(usize, Option<usize>)> {
let conn = self.reader();
let chunked: i64 = conn.query_row(
"SELECT COUNT(DISTINCT c.memory_id) FROM memory_chunks c \
JOIN memories m ON m.id = c.memory_id \
WHERE m.tier = 'hot' AND m.superseded_by IS NULL",
[],
|r| r.get(0),
)?;
let pending: Option<String> = conn
.query_row(
"SELECT value FROM index_meta WHERE key = 'chunks_pending'",
[],
|r| r.get(0),
)
.optional()?;
Ok((chunked as usize, pending.and_then(|v| v.parse().ok())))
}
pub(crate) fn set_chunks_pending(&self, pending: usize) -> Result<()> {
let conn = self.conn();
conn.execute(
"INSERT OR REPLACE INTO index_meta (key, value) VALUES ('chunks_pending', ?1)",
params![pending.to_string()],
)?;
Ok(())
}
pub fn store_chunks(
&self,
memory_id: Uuid,
content_hash: &str,
chunks: &[(usize, String, Vec<f32>)],
) -> Result<bool> {
let id = memory_id.to_string();
self.with_txn(|tx| {
let content: Option<String> = tx
.query_row(
"SELECT content FROM memories \
WHERE id = ?1 AND tier = 'hot' AND superseded_by IS NULL",
params![id],
|r| r.get(0),
)
.optional()?;
if content.is_none_or(|c| chunk::content_hash(&c) != content_hash) {
return Ok(false);
}
tx.execute(
"DELETE FROM memory_chunks WHERE memory_id = ?1",
params![id],
)?;
let mut insert = tx.prepare(
"INSERT INTO memory_chunks (memory_id, ord, text, embedding, content_hash) \
VALUES (?1, ?2, ?3, ?4, ?5)",
)?;
for (ord, text, embedding) in chunks {
let blob = (!embedding.is_empty()).then(|| embedding_to_blob(embedding));
insert.execute(params![id, *ord as i64, text, blob, content_hash])?;
}
Self::append_outbox(tx, memory_id, "upsert")?;
Ok(true)
})
}
pub fn chunks_for(&self, memory_id: Uuid) -> Result<Vec<MemoryChunk>> {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT ord, text, embedding, content_hash FROM memory_chunks \
WHERE memory_id = ?1 ORDER BY ord",
)?;
let rows = stmt.query_map(params![memory_id.to_string()], |r| {
Ok(MemoryChunk {
ord: r.get::<_, i64>(0)? as usize,
text: r.get(1)?,
embedding: r
.get::<_, Option<Vec<u8>>>(2)?
.map(|b| blob_to_embedding(&b))
.unwrap_or_default(),
content_hash: r.get(3)?,
})
})?;
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
}
pub fn delete_chunks(&self, memory_id: Uuid) -> Result<()> {
let conn = self.conn();
conn.execute(
"DELETE FROM memory_chunks WHERE memory_id = ?1",
params![memory_id.to_string()],
)?;
Ok(())
}
pub fn chunk_vec_key(&self, memory_id: Uuid, ord: usize) -> Result<u64> {
let conn = self.conn();
let id = memory_id.to_string();
conn.execute(
"INSERT OR IGNORE INTO chunk_vec_keys (memory_id, ord) VALUES (?1, ?2)",
params![id, ord as i64],
)?;
let key: i64 = conn.query_row(
"SELECT key FROM chunk_vec_keys WHERE memory_id = ?1 AND ord = ?2",
params![id, ord as i64],
|r| r.get(0),
)?;
Ok(key as u64)
}
pub fn chunk_vec_keys_for(&self, memory_id: Uuid) -> Result<Vec<(usize, u64)>> {
let conn = self.reader();
let mut stmt =
conn.prepare("SELECT ord, key FROM chunk_vec_keys WHERE memory_id = ?1 ORDER BY ord")?;
let rows = stmt.query_map(params![memory_id.to_string()], |r| {
Ok((r.get::<_, i64>(0)? as usize, r.get::<_, i64>(1)? as u64))
})?;
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
}
pub fn delete_chunk_vec_key(&self, memory_id: Uuid, ord: usize) -> Result<()> {
let conn = self.conn();
conn.execute(
"DELETE FROM chunk_vec_keys WHERE memory_id = ?1 AND ord = ?2",
params![memory_id.to_string(), ord as i64],
)?;
Ok(())
}
pub fn chunk_for_vec_key(&self, key: u64) -> Result<Option<(Uuid, usize)>> {
let conn = self.reader();
let hit: Option<(String, i64)> = conn
.query_row(
"SELECT memory_id, ord FROM chunk_vec_keys WHERE key = ?1",
params![key as i64],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.optional()?;
hit.map(|(id, ord)| Ok((Uuid::parse_str(&id)?, ord as usize)))
.transpose()
}
pub fn embedded_chunk_count(&self, dim: usize) -> Result<usize> {
let conn = self.reader();
let count: i64 = conn.query_row(
"SELECT COUNT(*) FROM memory_chunks c JOIN memories m ON m.id = c.memory_id \
WHERE m.tier = 'hot' AND m.superseded_by IS NULL \
AND c.embedding IS NOT NULL AND length(c.embedding) = ?1",
params![(dim * 4) as i64],
|r| r.get(0),
)?;
Ok(count as usize)
}
pub fn live_chunked_memories(&self) -> Result<Vec<(Uuid, String)>> {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT m.id, m.content FROM memories m \
WHERE m.tier = 'hot' AND m.superseded_by IS NULL \
AND EXISTS (SELECT 1 FROM memory_chunks c WHERE c.memory_id = m.id)",
)?;
let rows = stmt.query_map([], |r| {
let id = parse_uuid(&r.get::<_, String>(0)?)?;
let content: String = r.get(1)?;
Ok((id, chunk::content_hash(&content)))
})?;
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
}
pub fn purge_chunks_outside_live_set(&self) -> Result<()> {
const LIVE: &str = "SELECT id FROM memories WHERE tier = 'hot' AND superseded_by IS NULL";
self.with_txn(|tx| {
tx.execute(
&format!("DELETE FROM memory_chunks WHERE memory_id NOT IN ({LIVE})"),
[],
)?;
tx.execute(
&format!("DELETE FROM chunk_vec_keys WHERE memory_id NOT IN ({LIVE})"),
[],
)?;
Ok(())
})
}
pub(crate) fn chunk_window_text(&self, memory_id: Uuid, ord: usize) -> Result<Option<String>> {
let conn = self.reader();
let mut stmt = conn.prepare(
"SELECT text FROM memory_chunks WHERE memory_id = ?1 AND ord BETWEEN ?2 AND ?3 \
ORDER BY ord",
)?;
let texts = stmt
.query_map(
params![
memory_id.to_string(),
ord.saturating_sub(1) as i64,
ord as i64 + 1
],
|r| r.get::<_, String>(0),
)?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok((!texts.is_empty()).then(|| texts.join(" ")))
}
pub fn filtered_chunk_search(
&self,
embedding: &[f32],
source: Option<&str>,
memory_type: Option<MemoryType>,
project_path: Option<&str>,
limit: usize,
) -> Result<Vec<(Uuid, f32, usize)>> {
if limit == 0 {
return Ok(Vec::new());
}
let (predicate, args) = Self::scope_predicate(source, memory_type, project_path);
let conn = self.reader();
let mut stmt = conn.prepare(&format!(
"SELECT memory_id, ord, embedding FROM memory_chunks \
WHERE embedding IS NOT NULL AND memory_id IN ( \
SELECT id FROM memories \
WHERE tier = 'hot' AND superseded_by IS NULL{predicate}) \
ORDER BY memory_id, ord"
))?;
let bound: Vec<&dyn rusqlite::ToSql> =
args.iter().map(|a| a as &dyn rusqlite::ToSql).collect();
let query_norm = embedding.iter().map(|x| x * x).sum::<f32>().sqrt();
let mut rows = stmt.query(bound.as_slice())?;
let mut top: Vec<(f32, String, usize)> = Vec::with_capacity(limit + 1);
let mut current: Option<(f32, String, usize)> = None;
let place = |best: (f32, String, usize), top: &mut Vec<(f32, String, usize)>| {
let at = top.partition_point(|(s, _, _)| *s >= best.0);
if at < limit {
top.insert(at, best);
top.truncate(limit);
}
};
while let Some(row) = rows.next()? {
let Some(blob) = row.get_ref(2)?.as_blob_or_null()? else {
continue;
};
let similarity = cosine_blob(embedding, query_norm, blob);
let memory_id = row.get_ref(0)?.as_str()?;
let ord = row.get::<_, i64>(1)? as usize;
match &mut current {
Some(best) if best.1 == memory_id => {
if similarity > best.0 {
best.0 = similarity;
best.2 = ord;
}
}
_ => {
if let Some(done) = current.take() {
place(done, &mut top);
}
current = Some((similarity, memory_id.to_string(), ord));
}
}
}
if let Some(done) = current.take() {
place(done, &mut top);
}
top.into_iter()
.map(|(similarity, id, ord)| Ok((Uuid::parse_str(&id)?, similarity, ord)))
.collect()
}
}