use std::collections::HashSet;
use anyhow::Result;
use uuid::Uuid;
use super::Indexes;
use crate::sqlite_store::SqliteStore;
const PAGE: usize = 1_000;
impl Indexes {
pub fn drain(&mut self, store: &SqliteStore) -> Result<usize> {
self.require_writable()?;
let watermark = store.index_watermark()?;
let mut last = watermark;
let mut applied = 0;
loop {
let rows = store.outbox_page(last, PAGE)?;
let Some(page_last) = rows.last().map(|r| r.seq) else {
break;
};
applied += rows.len();
for row in rows {
self.apply(store, row.memory_id)?;
}
last = page_last;
}
if applied == 0 {
return Ok(0);
}
self.persist()?;
store.set_index_watermark(last)?;
store.prune_outbox(last)?;
Ok(applied)
}
fn apply(&mut self, store: &SqliteStore, memory_id: Uuid) -> Result<()> {
match store.memory_for_indexing(memory_id)? {
Some(row) if row.hot => {
if row.embedding.len() == super::EMBEDDING_DIM {
let key = store.vec_key(memory_id)?;
self.vector_mut().upsert(key, &row.embedding)?;
} else if let Some(key) = store.existing_vec_key(memory_id)? {
self.vector_mut().remove(key)?;
store.delete_vec_key(memory_id)?;
}
self.lexical_mut().upsert(memory_id, &row.body)?;
self.apply_chunks(store, memory_id, Some(&row.content_hash))
}
_ => {
if let Some(key) = store.existing_vec_key(memory_id)? {
self.vector_mut().remove(key)?;
store.delete_vec_key(memory_id)?;
}
self.lexical_mut().remove(memory_id)?;
self.apply_chunks(store, memory_id, None)
}
}
}
fn apply_chunks(
&mut self,
store: &SqliteStore,
memory_id: Uuid,
live_hash: Option<&str>,
) -> Result<()> {
let chunks = store.chunks_for(memory_id)?;
let fresh =
live_hash.is_some_and(|hash| chunks.first().is_some_and(|c| c.content_hash == hash));
if !fresh && !chunks.is_empty() {
store.delete_chunks(memory_id)?;
}
let mut indexed = HashSet::new();
if fresh {
for chunk in chunks
.iter()
.filter(|c| c.embedding.len() == super::EMBEDDING_DIM)
{
let key = store.chunk_vec_key(memory_id, chunk.ord)?;
self.chunks_mut().upsert(key, &chunk.embedding)?;
indexed.insert(chunk.ord);
}
}
for (ord, key) in store.chunk_vec_keys_for(memory_id)? {
if !indexed.contains(&ord) {
self.chunks_mut().remove(key)?;
store.delete_chunk_vec_key(memory_id, ord)?;
}
}
Ok(())
}
pub fn rebuild(&mut self, store: &SqliteStore) -> Result<usize> {
self.require_writable()?;
let snapshot_seq = store.max_outbox_seq()?;
let count = if self.vector.has_shared_readers() || self.chunks.has_shared_readers() {
self.rebuild_beside_readers(store)?
} else {
self.vector = self.vector.detached()?;
self.chunks = self.chunks.detached()?;
self.unpersisted = true;
self.fill_or_roll_back(store)?
};
self.persist()?;
store.set_index_watermark(snapshot_seq)?;
store.prune_outbox(snapshot_seq)?;
Ok(count)
}
fn rebuild_beside_readers(&mut self, store: &SqliteStore) -> Result<usize> {
let fresh_vector = self.vector.detached()?;
let fresh_chunks = self.chunks.detached()?;
let live_vector = std::mem::replace(&mut self.vector, fresh_vector);
let live_chunks = std::mem::replace(&mut self.chunks, fresh_chunks);
let built = self.fill_or_roll_back(store);
let fresh_vector = std::mem::replace(&mut self.vector, live_vector);
let fresh_chunks = std::mem::replace(&mut self.chunks, live_chunks);
let count = built?;
self.vector_mut().adopt(fresh_vector)?;
self.chunks_mut().adopt(fresh_chunks)?;
Ok(count)
}
fn fill_or_roll_back(&mut self, store: &SqliteStore) -> Result<usize> {
self.fill_from_store(store)
.or_else(|e| match self.lexical.rollback() {
Ok(()) => Err(e),
Err(rollback) => Err(e.context(format!(
"rolling back the lexical rebuild also failed: {rollback:#}"
))),
})
}
fn fill_from_store(&mut self, store: &SqliteStore) -> Result<usize> {
self.lexical_mut().reset()?;
let mut count = 0;
let mut after = 0;
loop {
let rows = store.indexable_page(after, PAGE)?;
let Some(last) = rows.last().map(|(rowid, _, _, _)| *rowid) else {
break;
};
count += rows.len();
for (_, memory_id, embedding, body) in rows {
if embedding.len() == super::EMBEDDING_DIM {
let key = store.vec_key(memory_id)?;
self.vector_mut().upsert(key, &embedding)?;
}
self.lexical_mut().upsert(memory_id, &body)?;
}
after = last;
}
store.purge_chunks_outside_live_set()?;
for (memory_id, content_hash) in store.live_chunked_memories()? {
self.apply_chunks(store, memory_id, Some(&content_hash))?;
}
Ok(count)
}
}