use std::collections::HashSet;
use anyhow::Result;
use uuid::Uuid;
use super::Indexes;
use crate::sqlite_store::SqliteStore;
impl Indexes {
pub fn drain(&mut self, store: &SqliteStore) -> Result<usize> {
self.require_writable()?;
let watermark = store.index_watermark()?;
let rows = store.outbox_after(watermark)?;
let Some(last) = rows.last().map(|r| r.seq) else {
return Ok(0);
};
let applied = rows.len();
for row in rows {
self.apply(store, row.memory_id)?;
}
self.persist()?;
store.set_index_watermark(last)?;
store.prune_outbox(last)?;
Ok(applied)
}
fn apply(&mut self, store: &SqliteStore, memory_id: Uuid) -> Result<()> {
match store.memory_for_indexing(memory_id)? {
Some(row) if row.hot => {
if row.embedding.len() == super::EMBEDDING_DIM {
let key = store.vec_key(memory_id)?;
self.vector_mut().upsert(key, &row.embedding)?;
} else if let Some(key) = store.existing_vec_key(memory_id)? {
self.vector_mut().remove(key)?;
store.delete_vec_key(memory_id)?;
}
self.lexical_mut().upsert(memory_id, &row.body)?;
self.apply_chunks(store, memory_id, Some(&row.content_hash))
}
_ => {
if let Some(key) = store.existing_vec_key(memory_id)? {
self.vector_mut().remove(key)?;
store.delete_vec_key(memory_id)?;
}
self.lexical_mut().remove(memory_id)?;
self.apply_chunks(store, memory_id, None)
}
}
}
fn apply_chunks(
&mut self,
store: &SqliteStore,
memory_id: Uuid,
live_hash: Option<&str>,
) -> Result<()> {
let chunks = store.chunks_for(memory_id)?;
let fresh =
live_hash.is_some_and(|hash| chunks.first().is_some_and(|c| c.content_hash == hash));
if !fresh && !chunks.is_empty() {
store.delete_chunks(memory_id)?;
}
let mut indexed = HashSet::new();
if fresh {
for chunk in chunks
.iter()
.filter(|c| c.embedding.len() == super::EMBEDDING_DIM)
{
let key = store.chunk_vec_key(memory_id, chunk.ord)?;
self.chunks_mut().upsert(key, &chunk.embedding)?;
indexed.insert(chunk.ord);
}
}
for (ord, key) in store.chunk_vec_keys_for(memory_id)? {
if !indexed.contains(&ord) {
self.chunks_mut().remove(key)?;
store.delete_chunk_vec_key(memory_id, ord)?;
}
}
Ok(())
}
pub fn rebuild(&mut self, store: &SqliteStore) -> Result<usize> {
self.require_writable()?;
let snapshot_seq = store.max_outbox_seq()?;
self.vector_mut().reset()?;
self.chunks_mut().reset()?;
self.lexical_mut().reset()?;
let rows = store.all_indexable()?;
let count = rows.len();
for (memory_id, embedding, body) in rows {
if embedding.len() == super::EMBEDDING_DIM {
let key = store.vec_key(memory_id)?;
self.vector_mut().upsert(key, &embedding)?;
}
self.lexical_mut().upsert(memory_id, &body)?;
}
store.purge_chunks_outside_live_set()?;
for (memory_id, content_hash) in store.live_chunked_memories()? {
self.apply_chunks(store, memory_id, Some(&content_hash))?;
}
self.persist()?;
store.set_index_watermark(snapshot_seq)?;
store.prune_outbox(snapshot_seq)?;
Ok(count)
}
}