second-brain-core 0.12.0

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::collections::HashMap;

use anyhow::Result;
use chrono::{DateTime, Utc};
use rusqlite::params;
use uuid::Uuid;

use super::sessions::session_of;
use super::sync_apply::UNCONSOLIDATED_PREDICATE;
use super::{SqliteStore, parse_dt, parse_uuid};
use crate::schema::AGENT_NOTE_SOURCE;
use crate::store::StoreAdmin;

const CLAUDE_CODE_SOURCE: &str = "claude-code";

/// The `distillation_log` status of a raw retired because an agent note
/// covers its session, so no LLM ever saw it.
pub const NOTE_COVERED: &str = "note-covered";

impl SqliteStore {
    /// Marks every undistilled Claude Code raw that a live agent note of its
    /// session covers, meaning the raw is timed no later than the session's
    /// newest live note, then archives the retirable raws. A raw is timed by
    /// `session_at`, else `created_at`, and so is a note. Returns how many
    /// raws were marked.
    pub fn retire_note_covered_raws(&self) -> Result<usize> {
        const BATCH: usize = 500;
        let covered = self.note_covered_candidates()?;
        for chunk in covered.chunks(BATCH) {
            self.with_txn(|tx| {
                for &id in chunk {
                    self.mark_distillation_on_tx(tx, id, id, AGENT_NOTE_SOURCE, NOTE_COVERED)?;
                }
                Ok(())
            })?;
        }
        self.retire_consolidated_raws()?;
        Ok(covered.len())
    }

    /// Whether an LLM distilled any raw at or after `cutoff`, on this
    /// machine or on a peer, since distillation markers replicate with
    /// their original time. Markers written by note cover are not LLM work,
    /// nor are the `legacy` copies repair gives their exact-text twins, which
    /// keep the `agent-note` model.
    pub fn llm_distilled_since(&self, cutoff: DateTime<Utc>) -> Result<bool> {
        Ok(self.reader().query_row(
            "SELECT EXISTS (SELECT 1 FROM distillation_log \
             WHERE status != ?1 AND COALESCE(model, '') != ?2 AND distilled_at >= ?3)",
            params![NOTE_COVERED, AGENT_NOTE_SOURCE, cutoff.to_rfc3339()],
            |r| r.get(0),
        )?)
    }

    fn note_covered_candidates(&self) -> Result<Vec<Uuid>> {
        let conn = self.reader();
        let mut newest_note: HashMap<String, DateTime<Utc>> = HashMap::new();
        {
            let mut stmt = conn.prepare(
                "SELECT source_id, COALESCE(session_at, created_at) FROM memories \
                 WHERE source = ?1 AND tier = 'hot' AND superseded_by IS NULL",
            )?;
            let rows = stmt.query_map(params![AGENT_NOTE_SOURCE], |r| {
                Ok((r.get::<_, String>(0)?, parse_dt(&r.get::<_, String>(1)?)?))
            })?;
            for row in rows {
                let (session, at) = row?;
                let newest = newest_note.entry(session).or_insert(at);
                *newest = (*newest).max(at);
            }
        }
        if newest_note.is_empty() {
            return Ok(Vec::new());
        }
        let mut stmt = conn.prepare(&format!(
            "SELECT id, source_id, COALESCE(session_at, created_at) FROM memories \
             WHERE source = ?1 AND {UNCONSOLIDATED_PREDICATE} ORDER BY created_at, id"
        ))?;
        let rows = stmt.query_map(params![CLAUDE_CODE_SOURCE], |r| {
            Ok((
                parse_uuid(&r.get::<_, String>(0)?)?,
                r.get::<_, String>(1)?,
                parse_dt(&r.get::<_, String>(2)?)?,
            ))
        })?;
        let mut covered = Vec::new();
        for row in rows {
            let (id, source_id, at) = row?;
            let note_at = session_of(&source_id).and_then(|s| newest_note.get(&s));
            if note_at.is_some_and(|&note_at| at <= note_at) {
                covered.push(id);
            }
        }
        Ok(covered)
    }
}

#[cfg(test)]
mod tests {
    use chrono::{Duration, TimeZone};

    use super::*;
    use crate::schema::{Memory, MemoryType, SyncNodeType, SyncOp, Tier};
    use crate::store::Store;

    const PROJECTS: &str = "/home/someone/.claude/projects/-work-demo";

    fn store() -> SqliteStore {
        SqliteStore::in_memory("machine-a".to_string()).unwrap()
    }

    fn at(hour: i64) -> DateTime<Utc> {
        Utc.with_ymd_and_hms(2026, 7, 1, 0, 0, 0).unwrap() + Duration::hours(hour)
    }

    fn raw_timed(
        s: &SqliteStore,
        source: &str,
        source_id: &str,
        session_at: Option<DateTime<Utc>>,
        created_at: DateTime<Utc>,
    ) -> Memory {
        let mut m = Memory::new(
            format!("a chunk of {source_id}"),
            MemoryType::Episodic,
            source.to_string(),
            source_id.to_string(),
        );
        m.session_at = session_at;
        m.created_at = created_at;
        s.store_memory(&m).unwrap();
        m
    }

    fn raw(s: &SqliteStore, source: &str, source_id: &str) -> Memory {
        raw_timed(s, source, source_id, None, at(9))
    }

    fn transcript_path(session: &str) -> String {
        format!("{PROJECTS}/{session}.jsonl")
    }

    fn transcript(s: &SqliteStore, session: &str) -> Memory {
        raw(s, "claude-code", &transcript_path(session))
    }

    fn note_timed(
        s: &SqliteStore,
        session: &str,
        session_at: Option<DateTime<Utc>>,
        created_at: DateTime<Utc>,
    ) -> Memory {
        let mut m = Memory::new(
            "Session-end note: kept the cache per worktree".to_string(),
            MemoryType::Decision,
            AGENT_NOTE_SOURCE.to_string(),
            session.to_string(),
        );
        m.session_at = session_at;
        m.created_at = created_at;
        s.store_memory(&m).unwrap();
        m
    }

    fn note(s: &SqliteStore, session: &str) -> Memory {
        note_timed(s, session, None, at(12))
    }

    fn marked(s: &SqliteStore, raw: Uuid, status: &str, distilled_at: DateTime<Utc>) {
        marked_by(s, raw, status, "some-model", distilled_at);
    }

    fn marked_by(s: &SqliteStore, raw: Uuid, status: &str, model: &str, at: DateTime<Utc>) {
        let entry = crate::schema::Distillation {
            memory_id: raw,
            distilled_id: Some(raw),
            distilled_at: Some(at),
            model: Some(model.to_string()),
            status: status.to_string(),
        };
        s.with_txn(|tx| SqliteStore::insert_distillation(tx, &entry))
            .unwrap();
    }

    fn marker(s: &SqliteStore, id: Uuid) -> Option<(String, String, String)> {
        s.reader()
            .query_row(
                "SELECT distilled_id, model, status FROM distillation_log WHERE memory_id = ?1",
                params![id.to_string()],
                |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
            )
            .ok()
    }

    fn tier(s: &SqliteStore, id: Uuid) -> Tier {
        s.get_memory(id).unwrap().unwrap().tier
    }

    #[test]
    fn a_raw_whose_session_has_a_note_is_marked_and_archived() {
        let s = store();
        note(&s, "sess-1");
        let covered = transcript(&s, "sess-1");
        let subagent = raw(
            &s,
            "claude-code",
            &format!("{PROJECTS}/sess-1/subagents/agent-7.jsonl"),
        );

        assert_eq!(s.retire_note_covered_raws().unwrap(), 2);

        for id in [covered.id, subagent.id] {
            assert_eq!(
                marker(&s, id),
                Some((
                    id.to_string(),
                    "agent-note".to_string(),
                    NOTE_COVERED.to_string()
                ))
            );
            assert_eq!(tier(&s, id), Tier::Archived);
            let replicated = s
                .sync_log_since(0)
                .unwrap()
                .into_iter()
                .filter(|e| {
                    e.node_type == SyncNodeType::Distillation
                        && e.op == SyncOp::Create
                        && e.node_id == id.to_string()
                })
                .count();
            assert_eq!(replicated, 1, "peers learn the raw is spoken for");
        }
    }

    #[test]
    fn raws_without_a_live_note_for_their_session_stay_hot_and_unmarked() {
        let s = store();
        note(&s, "sess-1");
        let other_session = transcript(&s, "sess-2");

        let superseded = note(&s, "sess-3");
        let newer = note(&s, "sess-other");
        s.supersede_memory(superseded.id, newer.id).unwrap();
        let of_superseded = transcript(&s, "sess-3");

        let archived = note(&s, "sess-4");
        s.archive_memory(archived.id).unwrap();
        let of_archived = transcript(&s, "sess-4");

        let not_claude = raw(&s, "codex", &format!("{PROJECTS}/sess-1.jsonl"));

        assert_eq!(s.retire_note_covered_raws().unwrap(), 0);
        for id in [
            other_session.id,
            of_superseded.id,
            of_archived.id,
            not_claude.id,
        ] {
            assert_eq!(marker(&s, id), None);
            assert_eq!(tier(&s, id), Tier::Hot);
        }
    }

    #[test]
    fn a_second_pass_marks_nothing() {
        let s = store();
        note(&s, "sess-1");
        transcript(&s, "sess-1");
        assert_eq!(s.retire_note_covered_raws().unwrap(), 1);
        let log = s.sync_log_since(0).unwrap().len();

        assert_eq!(s.retire_note_covered_raws().unwrap(), 0);
        assert_eq!(s.sync_log_since(0).unwrap().len(), log);
    }

    #[test]
    fn a_raw_the_llm_already_distilled_is_not_remarked() {
        let s = store();
        note(&s, "sess-1");
        let distilled = transcript(&s, "sess-1");
        let summary = raw(&s, "consolidation", "batch-1");
        s.mark_consolidated(distilled.id, summary.id, "some-model")
            .unwrap();

        assert_eq!(s.retire_note_covered_raws().unwrap(), 0);
        assert_eq!(marker(&s, distilled.id).unwrap().2, "legacy");
    }

    #[test]
    fn only_raws_no_later_than_the_newest_live_note_of_their_session_are_retired() {
        let s = store();
        let path = transcript_path("sess-1");
        note_timed(&s, "sess-1", None, at(10));
        note_timed(&s, "sess-1", Some(at(14)), at(30));
        let superseded = note_timed(&s, "sess-1", None, at(40));
        let newer = note(&s, "sess-other");
        s.supersede_memory(superseded.id, newer.id).unwrap();

        let before = raw_timed(&s, "claude-code", &path, Some(at(13)), at(50));
        let at_the_note = raw_timed(&s, "claude-code", &path, Some(at(14)), at(50));
        let created_before = raw_timed(&s, "claude-code", &path, None, at(11));
        let after = raw_timed(&s, "claude-code", &path, Some(at(15)), at(1));
        let created_after = raw_timed(&s, "claude-code", &path, None, at(20));

        assert_eq!(s.retire_note_covered_raws().unwrap(), 3);
        for id in [before.id, at_the_note.id, created_before.id] {
            assert_eq!(marker(&s, id).unwrap().2, NOTE_COVERED);
            assert_eq!(tier(&s, id), Tier::Archived);
        }
        for id in [after.id, created_after.id] {
            assert_eq!(marker(&s, id), None, "later work keeps its raw");
            assert_eq!(tier(&s, id), Tier::Hot);
        }
    }

    #[test]
    fn a_pass_retires_raws_an_earlier_pass_marked_but_did_not_archive() {
        let s = store();
        let marked_only = transcript(&s, "sess-1");
        marked(&s, marked_only.id, NOTE_COVERED, at(12));
        assert_eq!(tier(&s, marked_only.id), Tier::Hot);

        assert_eq!(s.retire_note_covered_raws().unwrap(), 0);
        assert_eq!(tier(&s, marked_only.id), Tier::Archived);
    }

    #[test]
    fn llm_distillation_counts_from_the_cutoff_and_note_cover_markers_do_not() {
        let s = store();
        let distilled = transcript(&s, "sess-1");
        let covered = transcript(&s, "sess-2");
        marked(&s, distilled.id, "legacy", at(100));
        marked(&s, covered.id, NOTE_COVERED, at(200));

        assert!(s.llm_distilled_since(at(100)).unwrap());
        assert!(s.llm_distilled_since(at(99)).unwrap());
        assert!(!s.llm_distilled_since(at(101)).unwrap());
        assert!(!store().llm_distilled_since(at(0)).unwrap());
    }

    #[test]
    fn a_note_cover_marker_inherited_by_a_twin_is_not_llm_work() {
        let s = store();
        let twin = transcript(&s, "sess-3");
        marked_by(&s, twin.id, "legacy", AGENT_NOTE_SOURCE, at(300));

        assert!(!s.llm_distilled_since(at(0)).unwrap());
    }
}