second-brain-core 0.13.2

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::collections::HashSet;

use anyhow::Result;
use uuid::Uuid;

use super::Indexes;
use crate::sqlite_store::SqliteStore;

/// Rows a drain or rebuild reads at a time, so a merged snapshot's
/// million-row outbox or store never sits in memory whole.
const PAGE: usize = 1_000;

impl Indexes {
    /// Apply every outbox row past the watermark to both indexes, persist
    /// them, then advance the watermark and prune. Persisting BEFORE the
    /// watermark moves gives at-least-once delivery across a crash; replay is
    /// safe because both index upserts are delete-then-add.
    pub fn drain(&mut self, store: &SqliteStore) -> Result<usize> {
        self.require_writable()?;
        let watermark = store.index_watermark()?;
        let mut last = watermark;
        let mut applied = 0;
        loop {
            let rows = store.outbox_page(last, PAGE)?;
            let Some(page_last) = rows.last().map(|r| r.seq) else {
                break;
            };
            applied += rows.len();
            for row in rows {
                self.apply(store, row.memory_id)?;
            }
            last = page_last;
        }
        if applied == 0 {
            return Ok(0);
        }
        self.persist()?;
        store.set_index_watermark(last)?;
        store.prune_outbox(last)?;
        Ok(applied)
    }

    /// The row's current store state decides the action, not the outbox op:
    /// a memory that is gone, superseded, archived, or embedding-less leaves
    /// the vector index, so stale ops replayed after a crash converge on the
    /// same end state.
    fn apply(&mut self, store: &SqliteStore, memory_id: Uuid) -> Result<()> {
        match store.memory_for_indexing(memory_id)? {
            Some(row) if row.hot => {
                if row.embedding.len() == super::EMBEDDING_DIM {
                    let key = store.vec_key(memory_id)?;
                    self.vector_mut().upsert(key, &row.embedding)?;
                } else if let Some(key) = store.existing_vec_key(memory_id)? {
                    self.vector_mut().remove(key)?;
                    store.delete_vec_key(memory_id)?;
                }
                self.lexical_mut().upsert(memory_id, &row.body)?;
                self.apply_chunks(store, memory_id, Some(&row.content_hash))
            }
            _ => {
                if let Some(key) = store.existing_vec_key(memory_id)? {
                    self.vector_mut().remove(key)?;
                    store.delete_vec_key(memory_id)?;
                }
                self.lexical_mut().remove(memory_id)?;
                self.apply_chunks(store, memory_id, None)
            }
        }
    }

    /// A chunk set survives only beside a live row whose current content it
    /// was split from (`live_hash`). Anything else is dropped here, which is
    /// the one place every exit from the live set (delete, retire, archive,
    /// supersede, a content change, a sync apply) is guaranteed to pass.
    /// Only chunks already embedded are indexed; the drain never embeds.
    fn apply_chunks(
        &mut self,
        store: &SqliteStore,
        memory_id: Uuid,
        live_hash: Option<&str>,
    ) -> Result<()> {
        let chunks = store.chunks_for(memory_id)?;
        let fresh =
            live_hash.is_some_and(|hash| chunks.first().is_some_and(|c| c.content_hash == hash));
        if !fresh && !chunks.is_empty() {
            store.delete_chunks(memory_id)?;
        }
        let mut indexed = HashSet::new();
        if fresh {
            for chunk in chunks
                .iter()
                .filter(|c| c.embedding.len() == super::EMBEDDING_DIM)
            {
                let key = store.chunk_vec_key(memory_id, chunk.ord)?;
                self.chunks_mut().upsert(key, &chunk.embedding)?;
                indexed.insert(chunk.ord);
            }
        }
        for (ord, key) in store.chunk_vec_keys_for(memory_id)? {
            if !indexed.contains(&ord) {
                self.chunks_mut().remove(key)?;
                store.delete_chunk_vec_key(memory_id, ord)?;
            }
        }
        Ok(())
    }

    /// Recreate every index from the store alone. Embeddings are stored in
    /// SQLite precisely so this never re-embeds. The lexical index changes
    /// only at its commit, and while a `shared_reader` exists the vectors are
    /// built beside the live ones and swapped in whole, so it sees the old
    /// indexes or the new ones, never a part-built one.
    pub fn rebuild(&mut self, store: &SqliteStore) -> Result<usize> {
        self.require_writable()?;
        let snapshot_seq = store.max_outbox_seq()?;
        let count = if self.vector.has_shared_readers() || self.chunks.has_shared_readers() {
            self.rebuild_beside_readers(store)?
        } else {
            self.vector = self.vector.detached()?;
            self.chunks = self.chunks.detached()?;
            self.unpersisted = true;
            self.fill_or_roll_back(store)?
        };
        self.persist()?;
        store.set_index_watermark(snapshot_seq)?;
        store.prune_outbox(snapshot_seq)?;
        Ok(count)
    }

    /// Builds the new vector indexes beside the ones shared readers search,
    /// then swaps them in, so recall never sees a half-built index; vector
    /// memory doubles for the rebuild's length.
    fn rebuild_beside_readers(&mut self, store: &SqliteStore) -> Result<usize> {
        let fresh_vector = self.vector.detached()?;
        let fresh_chunks = self.chunks.detached()?;
        let live_vector = std::mem::replace(&mut self.vector, fresh_vector);
        let live_chunks = std::mem::replace(&mut self.chunks, fresh_chunks);
        let built = self.fill_or_roll_back(store);
        let fresh_vector = std::mem::replace(&mut self.vector, live_vector);
        let fresh_chunks = std::mem::replace(&mut self.chunks, live_chunks);
        let count = built?;
        self.vector_mut().adopt(fresh_vector)?;
        self.chunks_mut().adopt(fresh_chunks)?;
        Ok(count)
    }

    fn fill_or_roll_back(&mut self, store: &SqliteStore) -> Result<usize> {
        self.fill_from_store(store)
            .or_else(|e| match self.lexical.rollback() {
                Ok(()) => Err(e),
                Err(rollback) => Err(e.context(format!(
                    "rolling back the lexical rebuild also failed: {rollback:#}"
                ))),
            })
    }

    fn fill_from_store(&mut self, store: &SqliteStore) -> Result<usize> {
        self.lexical_mut().reset()?;
        let mut count = 0;
        let mut after = 0;
        loop {
            let rows = store.indexable_page(after, PAGE)?;
            let Some(last) = rows.last().map(|(rowid, _, _, _)| *rowid) else {
                break;
            };
            count += rows.len();
            for (_, memory_id, embedding, body) in rows {
                if embedding.len() == super::EMBEDDING_DIM {
                    let key = store.vec_key(memory_id)?;
                    self.vector_mut().upsert(key, &embedding)?;
                }
                self.lexical_mut().upsert(memory_id, &body)?;
            }
            after = last;
        }
        store.purge_chunks_outside_live_set()?;
        for (memory_id, content_hash) in store.live_chunked_memories()? {
            self.apply_chunks(store, memory_id, Some(&content_hash))?;
        }
        Ok(count)
    }
}