second-brain-core 0.10.0

Core library for second-brain: KuzuDB graph storage, BGE embeddings, and weighted query engine
Documentation
use std::collections::HashSet;

use anyhow::Result;
use uuid::Uuid;

use super::Indexes;
use crate::sqlite_store::SqliteStore;

impl Indexes {
    /// Apply every outbox row past the watermark to both indexes, persist
    /// them, then advance the watermark and prune. Persisting BEFORE the
    /// watermark moves gives at-least-once delivery across a crash; replay is
    /// safe because both index upserts are delete-then-add.
    pub fn drain(&mut self, store: &SqliteStore) -> Result<usize> {
        self.require_writable()?;
        let watermark = store.index_watermark()?;
        let rows = store.outbox_after(watermark)?;
        let Some(last) = rows.last().map(|r| r.seq) else {
            return Ok(0);
        };
        let applied = rows.len();
        for row in rows {
            self.apply(store, row.memory_id)?;
        }
        self.persist()?;
        store.set_index_watermark(last)?;
        store.prune_outbox(last)?;
        Ok(applied)
    }

    /// The row's current store state decides the action, not the outbox op:
    /// a memory that is gone, superseded, archived, or embedding-less leaves
    /// the vector index, so stale ops replayed after a crash converge on the
    /// same end state.
    fn apply(&mut self, store: &SqliteStore, memory_id: Uuid) -> Result<()> {
        match store.memory_for_indexing(memory_id)? {
            Some(row) if row.hot => {
                if row.embedding.len() == super::EMBEDDING_DIM {
                    let key = store.vec_key(memory_id)?;
                    self.vector_mut().upsert(key, &row.embedding)?;
                } else if let Some(key) = store.existing_vec_key(memory_id)? {
                    self.vector_mut().remove(key)?;
                    store.delete_vec_key(memory_id)?;
                }
                self.lexical_mut().upsert(memory_id, &row.body)?;
                self.apply_chunks(store, memory_id, Some(&row.content_hash))
            }
            _ => {
                if let Some(key) = store.existing_vec_key(memory_id)? {
                    self.vector_mut().remove(key)?;
                    store.delete_vec_key(memory_id)?;
                }
                self.lexical_mut().remove(memory_id)?;
                self.apply_chunks(store, memory_id, None)
            }
        }
    }

    /// A chunk set survives only beside a live row whose current content it
    /// was split from (`live_hash`). Anything else is dropped here, which is
    /// the one place every exit from the live set (delete, retire, archive,
    /// supersede, a content change, a sync apply) is guaranteed to pass.
    /// Only chunks already embedded are indexed; the drain never embeds.
    fn apply_chunks(
        &mut self,
        store: &SqliteStore,
        memory_id: Uuid,
        live_hash: Option<&str>,
    ) -> Result<()> {
        let chunks = store.chunks_for(memory_id)?;
        let fresh =
            live_hash.is_some_and(|hash| chunks.first().is_some_and(|c| c.content_hash == hash));
        if !fresh && !chunks.is_empty() {
            store.delete_chunks(memory_id)?;
        }
        let mut indexed = HashSet::new();
        if fresh {
            for chunk in chunks
                .iter()
                .filter(|c| c.embedding.len() == super::EMBEDDING_DIM)
            {
                let key = store.chunk_vec_key(memory_id, chunk.ord)?;
                self.chunks_mut().upsert(key, &chunk.embedding)?;
                indexed.insert(chunk.ord);
            }
        }
        for (ord, key) in store.chunk_vec_keys_for(memory_id)? {
            if !indexed.contains(&ord) {
                self.chunks_mut().remove(key)?;
                store.delete_chunk_vec_key(memory_id, ord)?;
            }
        }
        Ok(())
    }

    /// Recreate every index from the store alone. Embeddings are stored in
    /// SQLite precisely so this never re-embeds.
    pub fn rebuild(&mut self, store: &SqliteStore) -> Result<usize> {
        self.require_writable()?;
        let snapshot_seq = store.max_outbox_seq()?;
        self.vector_mut().reset()?;
        self.chunks_mut().reset()?;
        self.lexical_mut().reset()?;
        let rows = store.all_indexable()?;
        let count = rows.len();
        for (memory_id, embedding, body) in rows {
            if embedding.len() == super::EMBEDDING_DIM {
                let key = store.vec_key(memory_id)?;
                self.vector_mut().upsert(key, &embedding)?;
            }
            self.lexical_mut().upsert(memory_id, &body)?;
        }
        store.purge_chunks_outside_live_set()?;
        for (memory_id, content_hash) in store.live_chunked_memories()? {
            self.apply_chunks(store, memory_id, Some(&content_hash))?;
        }
        self.persist()?;
        store.set_index_watermark(snapshot_seq)?;
        store.prune_outbox(snapshot_seq)?;
        Ok(count)
    }
}