second-brain-core 0.10.0

Core library for second-brain: KuzuDB graph storage, BGE embeddings, and weighted query engine
Documentation
use anyhow::Result;
use rusqlite::{OptionalExtension, params};
use uuid::Uuid;

use super::{SqliteStore, blob_to_embedding, cosine_blob, embedding_to_blob, parse_uuid};
use crate::chunk;
use crate::schema::MemoryType;

/// A live memory whose stored chunk set is missing or no longer matches its
/// content: the chunks to embed and the hash `store_chunks` must be handed
/// back so a set split from since-changed content is refused.
#[derive(Debug, Clone, PartialEq)]
pub struct ChunkWork {
    pub memory_id: Uuid,
    pub content_hash: String,
    pub chunks: Vec<String>,
}

#[derive(Debug, Clone, PartialEq)]
pub struct MemoryChunk {
    pub ord: usize,
    pub text: String,
    pub embedding: Vec<f32>,
    pub content_hash: String,
}

impl SqliteStore {
    /// Live memories that split into two or more chunks and whose stored set
    /// is absent or was split from other content, oldest first. Splitting
    /// happens here in Rust, so every candidate row is split on each call;
    /// the length floor only skips rows too short to yield two chunks.
    pub fn chunks_needing_embedding(&self, limit: usize) -> Result<Vec<ChunkWork>> {
        Ok(self.scan_chunk_work(limit)?.0)
    }

    /// `chunks_needing_embedding`, plus the total number of memories needing
    /// chunks. Counting past `limit` keeps scanning the whole table but
    /// holds on to nothing beyond the first `limit`.
    pub fn scan_chunk_work(&self, limit: usize) -> Result<(Vec<ChunkWork>, usize)> {
        let conn = self.reader();
        let mut stmt = conn.prepare(
            "SELECT m.id, m.content, \
             (SELECT c.content_hash FROM memory_chunks c WHERE c.memory_id = m.id LIMIT 1) \
             FROM memories m \
             WHERE m.tier = 'hot' AND m.superseded_by IS NULL AND length(m.content) >= ?1 \
             ORDER BY m.rowid",
        )?;
        let mut rows = stmt.query(params![(2 * chunk::MIN_CHUNK_CHARS) as i64])?;
        let mut out = Vec::new();
        let mut pending = 0;
        while let Some(row) = rows.next()? {
            let content: String = row.get(1)?;
            let stored: Option<String> = row.get(2)?;
            let content_hash = chunk::content_hash(&content);
            if stored.as_deref() == Some(content_hash.as_str()) {
                continue;
            }
            let chunks = chunk::split(&content);
            if chunks.is_empty() {
                continue;
            }
            pending += 1;
            if out.len() < limit {
                out.push(ChunkWork {
                    memory_id: parse_uuid(&row.get::<_, String>(0)?)?,
                    content_hash,
                    chunks,
                });
            }
        }
        Ok((out, pending))
    }

    /// (live memories holding a chunk set, memories still needing one as of
    /// the last chunk backfill). The second is None before any backfill has
    /// run, because counting it fresh means splitting every live row.
    pub fn chunk_progress(&self) -> Result<(usize, Option<usize>)> {
        let conn = self.reader();
        let chunked: i64 = conn.query_row(
            "SELECT COUNT(DISTINCT c.memory_id) FROM memory_chunks c \
             JOIN memories m ON m.id = c.memory_id \
             WHERE m.tier = 'hot' AND m.superseded_by IS NULL",
            [],
            |r| r.get(0),
        )?;
        let pending: Option<String> = conn
            .query_row(
                "SELECT value FROM index_meta WHERE key = 'chunks_pending'",
                [],
                |r| r.get(0),
            )
            .optional()?;
        Ok((chunked as usize, pending.and_then(|v| v.parse().ok())))
    }

    pub(crate) fn set_chunks_pending(&self, pending: usize) -> Result<()> {
        let conn = self.conn();
        conn.execute(
            "INSERT OR REPLACE INTO index_meta (key, value) VALUES ('chunks_pending', ?1)",
            params![pending.to_string()],
        )?;
        Ok(())
    }

    /// Replaces the memory's whole chunk set and queues an index upsert for
    /// it, in one transaction. Returns false, writing nothing, when the
    /// memory is no longer live or its content no longer hashes to
    /// `content_hash`. Chunks are derived locally on every peer, so nothing
    /// here touches sync_log.
    pub fn store_chunks(
        &self,
        memory_id: Uuid,
        content_hash: &str,
        chunks: &[(usize, String, Vec<f32>)],
    ) -> Result<bool> {
        let id = memory_id.to_string();
        self.with_txn(|tx| {
            let content: Option<String> = tx
                .query_row(
                    "SELECT content FROM memories \
                     WHERE id = ?1 AND tier = 'hot' AND superseded_by IS NULL",
                    params![id],
                    |r| r.get(0),
                )
                .optional()?;
            if content.is_none_or(|c| chunk::content_hash(&c) != content_hash) {
                return Ok(false);
            }
            tx.execute(
                "DELETE FROM memory_chunks WHERE memory_id = ?1",
                params![id],
            )?;
            let mut insert = tx.prepare(
                "INSERT INTO memory_chunks (memory_id, ord, text, embedding, content_hash) \
                 VALUES (?1, ?2, ?3, ?4, ?5)",
            )?;
            for (ord, text, embedding) in chunks {
                let blob = (!embedding.is_empty()).then(|| embedding_to_blob(embedding));
                insert.execute(params![id, *ord as i64, text, blob, content_hash])?;
            }
            Self::append_outbox(tx, memory_id, "upsert")?;
            Ok(true)
        })
    }

    pub fn chunks_for(&self, memory_id: Uuid) -> Result<Vec<MemoryChunk>> {
        let conn = self.reader();
        let mut stmt = conn.prepare(
            "SELECT ord, text, embedding, content_hash FROM memory_chunks \
             WHERE memory_id = ?1 ORDER BY ord",
        )?;
        let rows = stmt.query_map(params![memory_id.to_string()], |r| {
            Ok(MemoryChunk {
                ord: r.get::<_, i64>(0)? as usize,
                text: r.get(1)?,
                embedding: r
                    .get::<_, Option<Vec<u8>>>(2)?
                    .map(|b| blob_to_embedding(&b))
                    .unwrap_or_default(),
                content_hash: r.get(3)?,
            })
        })?;
        Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
    }

    pub fn delete_chunks(&self, memory_id: Uuid) -> Result<()> {
        let conn = self.conn();
        conn.execute(
            "DELETE FROM memory_chunks WHERE memory_id = ?1",
            params![memory_id.to_string()],
        )?;
        Ok(())
    }

    /// Stable u64 key in `chunks.usearch` for one chunk, allocated on first
    /// use, mirroring `vec_key`.
    pub fn chunk_vec_key(&self, memory_id: Uuid, ord: usize) -> Result<u64> {
        let conn = self.conn();
        let id = memory_id.to_string();
        conn.execute(
            "INSERT OR IGNORE INTO chunk_vec_keys (memory_id, ord) VALUES (?1, ?2)",
            params![id, ord as i64],
        )?;
        let key: i64 = conn.query_row(
            "SELECT key FROM chunk_vec_keys WHERE memory_id = ?1 AND ord = ?2",
            params![id, ord as i64],
            |r| r.get(0),
        )?;
        Ok(key as u64)
    }

    /// (ord, key) pairs allocated for the memory, by ord.
    pub fn chunk_vec_keys_for(&self, memory_id: Uuid) -> Result<Vec<(usize, u64)>> {
        let conn = self.reader();
        let mut stmt =
            conn.prepare("SELECT ord, key FROM chunk_vec_keys WHERE memory_id = ?1 ORDER BY ord")?;
        let rows = stmt.query_map(params![memory_id.to_string()], |r| {
            Ok((r.get::<_, i64>(0)? as usize, r.get::<_, i64>(1)? as u64))
        })?;
        Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
    }

    /// Drops the key row once its vector is gone from `chunks.usearch`, so
    /// `chunk_vec_keys` only holds keys the index can still resolve.
    pub fn delete_chunk_vec_key(&self, memory_id: Uuid, ord: usize) -> Result<()> {
        let conn = self.conn();
        conn.execute(
            "DELETE FROM chunk_vec_keys WHERE memory_id = ?1 AND ord = ?2",
            params![memory_id.to_string(), ord as i64],
        )?;
        Ok(())
    }

    pub fn chunk_for_vec_key(&self, key: u64) -> Result<Option<(Uuid, usize)>> {
        let conn = self.reader();
        let hit: Option<(String, i64)> = conn
            .query_row(
                "SELECT memory_id, ord FROM chunk_vec_keys WHERE key = ?1",
                params![key as i64],
                |r| Ok((r.get(0)?, r.get(1)?)),
            )
            .optional()?;
        hit.map(|(id, ord)| Ok((Uuid::parse_str(&id)?, ord as usize)))
            .transpose()
    }

    /// Chunk rows of live memories whose embedding has exactly `dim`
    /// components: what `chunks.usearch` should hold once drained.
    pub fn embedded_chunk_count(&self, dim: usize) -> Result<usize> {
        let conn = self.reader();
        let count: i64 = conn.query_row(
            "SELECT COUNT(*) FROM memory_chunks c JOIN memories m ON m.id = c.memory_id \
             WHERE m.tier = 'hot' AND m.superseded_by IS NULL \
             AND c.embedding IS NOT NULL AND length(c.embedding) = ?1",
            params![(dim * 4) as i64],
            |r| r.get(0),
        )?;
        Ok(count as usize)
    }

    /// Live memories holding a chunk set, each with the hash of its current
    /// content, which the set must match to still be indexed.
    pub fn live_chunked_memories(&self) -> Result<Vec<(Uuid, String)>> {
        let conn = self.reader();
        let mut stmt = conn.prepare(
            "SELECT m.id, m.content FROM memories m \
             WHERE m.tier = 'hot' AND m.superseded_by IS NULL \
             AND EXISTS (SELECT 1 FROM memory_chunks c WHERE c.memory_id = m.id)",
        )?;
        let rows = stmt.query_map([], |r| {
            let id = parse_uuid(&r.get::<_, String>(0)?)?;
            let content: String = r.get(1)?;
            Ok((id, chunk::content_hash(&content)))
        })?;
        Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
    }

    /// Deletes chunk rows and chunk keys of every memory outside the live
    /// set. Only a full rebuild calls this, because it resets the chunk
    /// index first and so has no vectors to remove per key.
    pub fn purge_chunks_outside_live_set(&self) -> Result<()> {
        const LIVE: &str = "SELECT id FROM memories WHERE tier = 'hot' AND superseded_by IS NULL";
        self.with_txn(|tx| {
            tx.execute(
                &format!("DELETE FROM memory_chunks WHERE memory_id NOT IN ({LIVE})"),
                [],
            )?;
            tx.execute(
                &format!("DELETE FROM chunk_vec_keys WHERE memory_id NOT IN ({LIVE})"),
                [],
            )?;
            Ok(())
        })
    }

    pub(crate) fn chunk_window_text(&self, memory_id: Uuid, ord: usize) -> Result<Option<String>> {
        let conn = self.reader();
        let mut stmt = conn.prepare(
            "SELECT text FROM memory_chunks WHERE memory_id = ?1 AND ord BETWEEN ?2 AND ?3 \
             ORDER BY ord",
        )?;
        let texts = stmt
            .query_map(
                params![
                    memory_id.to_string(),
                    ord.saturating_sub(1) as i64,
                    ord as i64 + 1
                ],
                |r| r.get::<_, String>(0),
            )?
            .collect::<rusqlite::Result<Vec<_>>>()?;
        Ok((!texts.is_empty()).then(|| texts.join(" ")))
    }

    /// The scoped counterpart of `Indexes::chunk_search`: a brute-force
    /// cosine scan of the embedded chunks of live in-scope memories,
    /// returning each memory's best (memory id, cosine, ord) for the top
    /// `limit` memories, best first. Rows arrive grouped by memory (the
    /// primary key's order), so only the running best of one memory and the
    /// top `limit` are ever held, however wide the scope.
    pub fn filtered_chunk_search(
        &self,
        embedding: &[f32],
        source: Option<&str>,
        memory_type: Option<MemoryType>,
        project_path: Option<&str>,
        limit: usize,
    ) -> Result<Vec<(Uuid, f32, usize)>> {
        if limit == 0 {
            return Ok(Vec::new());
        }
        let (predicate, args) = Self::scope_predicate(source, memory_type, project_path);
        let conn = self.reader();
        let mut stmt = conn.prepare(&format!(
            "SELECT memory_id, ord, embedding FROM memory_chunks \
             WHERE embedding IS NOT NULL AND memory_id IN ( \
               SELECT id FROM memories \
               WHERE tier = 'hot' AND superseded_by IS NULL{predicate}) \
             ORDER BY memory_id, ord"
        ))?;
        let bound: Vec<&dyn rusqlite::ToSql> =
            args.iter().map(|a| a as &dyn rusqlite::ToSql).collect();
        let query_norm = embedding.iter().map(|x| x * x).sum::<f32>().sqrt();
        let mut rows = stmt.query(bound.as_slice())?;
        let mut top: Vec<(f32, String, usize)> = Vec::with_capacity(limit + 1);
        let mut current: Option<(f32, String, usize)> = None;
        let place = |best: (f32, String, usize), top: &mut Vec<(f32, String, usize)>| {
            let at = top.partition_point(|(s, _, _)| *s >= best.0);
            if at < limit {
                top.insert(at, best);
                top.truncate(limit);
            }
        };
        while let Some(row) = rows.next()? {
            let Some(blob) = row.get_ref(2)?.as_blob_or_null()? else {
                continue;
            };
            let similarity = cosine_blob(embedding, query_norm, blob);
            let memory_id = row.get_ref(0)?.as_str()?;
            let ord = row.get::<_, i64>(1)? as usize;
            match &mut current {
                Some(best) if best.1 == memory_id => {
                    if similarity > best.0 {
                        best.0 = similarity;
                        best.2 = ord;
                    }
                }
                _ => {
                    if let Some(done) = current.take() {
                        place(done, &mut top);
                    }
                    current = Some((similarity, memory_id.to_string(), ord));
                }
            }
        }
        if let Some(done) = current.take() {
            place(done, &mut top);
        }
        top.into_iter()
            .map(|(similarity, id, ord)| Ok((Uuid::parse_str(&id)?, similarity, ord)))
            .collect()
    }
}