second-brain-core 0.13.2

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::collections::HashMap;

use chrono::{DateTime, Duration, Utc};
use second_brain_core::{
    schema::{Conversation, Memory, MemoryType, Relation, RelationType, Tier},
    session_dates::{SessionEvidence, SessionTier},
    sqlite_store::SqliteStore,
    store::{Store, StoreAdmin},
};

#[derive(Default)]
struct Evidence {
    lines: HashMap<(String, String), DateTime<Utc>>,
    starts: HashMap<String, DateTime<Utc>>,
    mtimes: HashMap<String, DateTime<Utc>>,
}

impl SessionEvidence for Evidence {
    fn line_time(&self, transcript: &str, text: &str) -> Option<DateTime<Utc>> {
        self.lines
            .get(&(transcript.to_string(), text.to_string()))
            .copied()
    }
    fn first_time(&self, transcript: &str) -> Option<DateTime<Utc>> {
        self.starts.get(transcript).copied()
    }
    fn note_modified_at(&self, path: &str) -> Option<DateTime<Utc>> {
        self.mtimes.get(path).copied()
    }
}

fn at(s: &str) -> DateTime<Utc> {
    s.parse().unwrap()
}

fn mem(content: &str, source: &str, source_id: &str) -> Memory {
    Memory::new(
        content.to_string(),
        MemoryType::Semantic,
        source.to_string(),
        source_id.to_string(),
    )
}

fn derived_from(s: &SqliteStore, memory: &Memory, source: &str, started_at: DateTime<Utc>) {
    let conversation = Conversation {
        id: uuid::Uuid::new_v4(),
        source: source.to_string(),
        machine_id: "m".to_string(),
        started_at,
        project_path: None,
    };
    s.store_conversation(&conversation).unwrap();
    s.store_relation(&Relation {
        from_id: memory.id,
        to_id: conversation.id,
        relation_type: RelationType::DerivedFrom,
        strength: 1.0,
        context: None,
    })
    .unwrap();
}

struct Fixture {
    store: SqliteStore,
    evidence: Evidence,
    raw_one: Memory,
    distillate: Memory,
    conversation_raw: Memory,
    fallback_raw: Memory,
    note: Memory,
    edited_note: Memory,
    note_distillate: Memory,
    orphan_distillate: Memory,
    remembered: Memory,
    already: Memory,
}

fn fixture() -> Fixture {
    let store = SqliteStore::in_memory("m".to_string()).unwrap();
    let mut evidence = Evidence::default();
    let ingest_day = at("2026-08-20T12:00:00Z");

    let archived_raw = |content: &str, transcript: &str| {
        let mut m = mem(content, "claude-code", transcript);
        m.created_at = ingest_day;
        m.updated_at = ingest_day;
        m.tier = Tier::Archived;
        m
    };

    let raw_one = archived_raw("raw one from t1", "/t1.jsonl");
    let raw_two = archived_raw("raw two from t1", "/t1.jsonl");
    evidence.lines.insert(
        ("/t1.jsonl".into(), raw_one.content.clone()),
        at("2026-07-23T09:00:00Z"),
    );
    evidence.lines.insert(
        ("/t1.jsonl".into(), raw_two.content.clone()),
        at("2026-07-23T09:30:00Z"),
    );
    let distillate = mem("the fact distilled from t1", "consolidation", "/t1.jsonl");

    let conversation_raw = archived_raw("raw whose transcript is gone", "/gone.jsonl");
    let fallback_raw = archived_raw("raw of a conversation without timestamps", "/gone2.jsonl");

    let mut note = mem("a claude-memory note", "claude-memory", "/notes/n.md");
    note.created_at = ingest_day;
    evidence
        .mtimes
        .insert("/notes/n.md".into(), at("2026-06-01T08:00:00Z"));
    let mut edited_note = mem("a note edited after ingest", "claude-memory", "/notes/e.md");
    edited_note.created_at = ingest_day;
    evidence
        .mtimes
        .insert("/notes/e.md".into(), ingest_day + Duration::days(3));
    let mut note_distillate = mem("distilled from a note file", "consolidation", "/notes/d.md");
    note_distillate.created_at = ingest_day;
    evidence
        .mtimes
        .insert("/notes/d.md".into(), at("2026-04-04T04:00:00Z"));

    let orphan_distillate = mem(
        "distillate whose raws were purged",
        "consolidation",
        "/t6.jsonl",
    );
    evidence
        .starts
        .insert("/t6.jsonl".into(), at("2026-05-05T05:00:00Z"));

    let remembered = mem("saved with remember", "mcp-direct", "");

    let mut already = mem("dated before the backfill", "consolidation", "/t1.jsonl");
    already.session_at = Some(at("2026-01-01T00:00:00Z"));

    for m in [
        &raw_one,
        &raw_two,
        &distillate,
        &conversation_raw,
        &fallback_raw,
        &note,
        &edited_note,
        &note_distillate,
        &orphan_distillate,
        &remembered,
        &already,
    ] {
        store.store_memory(m).unwrap();
    }
    store
        .mark_consolidated(raw_one.id, distillate.id, "q")
        .unwrap();
    store
        .mark_consolidated(raw_two.id, distillate.id, "q")
        .unwrap();

    derived_from(
        &store,
        &conversation_raw,
        "claude-code",
        at("2026-07-01T10:00:00Z"),
    );
    // A conversation whose transcript had no timestamp was stamped with the
    // ingest clock, after its memories were created.
    derived_from(
        &store,
        &fallback_raw,
        "cursor",
        ingest_day + Duration::seconds(1),
    );
    // claude-memory conversations are always stamped now; the note's date
    // must come from its file instead.
    derived_from(&store, &note, "claude-memory", at("2026-02-02T00:00:00Z"));

    Fixture {
        store,
        evidence,
        raw_one,
        distillate,
        conversation_raw,
        fallback_raw,
        note,
        edited_note,
        note_distillate,
        orphan_distillate,
        remembered,
        already,
    }
}

fn session_at(s: &SqliteStore, m: &Memory) -> Option<DateTime<Utc>> {
    s.get_memory(m.id).unwrap().unwrap().session_at
}

#[test]
fn backfill_dates_each_memory_from_the_first_tier_with_evidence() {
    let f = fixture();
    let report = f
        .store
        .backfill_session_at(&f.evidence, false, false)
        .unwrap();

    assert_eq!(
        session_at(&f.store, &f.raw_one),
        Some(at("2026-07-23T09:00:00Z"))
    );
    assert_eq!(
        session_at(&f.store, &f.distillate),
        Some(at("2026-07-23T09:30:00Z")),
        "a distillate takes the latest of its raws"
    );
    assert_eq!(
        session_at(&f.store, &f.conversation_raw),
        Some(at("2026-07-01T10:00:00Z"))
    );
    assert_eq!(
        session_at(&f.store, &f.fallback_raw),
        None,
        "neither a fallback conversation nor an archived created_at is evidence"
    );
    assert_eq!(
        session_at(&f.store, &f.edited_note),
        None,
        "a note edited after the memory was made from it would date it too new"
    );
    assert_eq!(
        session_at(&f.store, &f.note_distillate),
        Some(at("2026-04-04T04:00:00Z")),
        "a distillate of a note file is dated by the unchanged file"
    );
    assert_eq!(
        session_at(&f.store, &f.note),
        Some(at("2026-06-01T08:00:00Z"))
    );
    assert_eq!(
        session_at(&f.store, &f.orphan_distillate),
        Some(at("2026-05-05T05:00:00Z"))
    );
    assert_eq!(
        session_at(&f.store, &f.remembered),
        Some(f.remembered.created_at)
    );
    assert_eq!(
        session_at(&f.store, &f.already),
        f.already.session_at,
        "a known date is never replaced"
    );

    let counts = &report.all;
    assert_eq!(counts.get(SessionTier::TranscriptLine), 3);
    assert_eq!(counts.get(SessionTier::ConversationStart), 1);
    assert_eq!(counts.get(SessionTier::TranscriptStart), 1);
    assert_eq!(counts.get(SessionTier::FileMtime), 2);
    assert_eq!(counts.get(SessionTier::RememberedAt), 1);
    assert_eq!(counts.get(SessionTier::Unknown), 2);
    assert_eq!(report.written, 8, "the unknown ones are left NULL");
    assert_eq!(report.fallback_conversations_skipped, 1);
    assert!(!report.sample.is_empty());
}

#[test]
fn backfill_leaves_updated_at_alone_logs_each_write_and_skips_the_index() {
    let f = fixture();
    let before: Vec<Memory> = f.store.all_memories_with_embeddings().unwrap();
    let log_before = f.store.max_sync_seq().unwrap();
    let outbox_before = f.store.max_outbox_seq().unwrap();

    let report = f
        .store
        .backfill_session_at(&f.evidence, false, false)
        .unwrap();

    for m in &before {
        assert_eq!(
            f.store.get_memory(m.id).unwrap().unwrap().updated_at,
            m.updated_at,
            "{}",
            m.content
        );
    }
    assert_eq!(
        f.store.sync_log_since(log_before).unwrap().len() as u64,
        report.written,
        "one sync Update per dated memory"
    );
    assert_eq!(f.store.max_outbox_seq().unwrap(), outbox_before);

    let again = f
        .store
        .backfill_session_at(&f.evidence, false, false)
        .unwrap();
    assert_eq!(again.written, 0, "a second run changes nothing");
    assert_eq!(
        f.store.sync_log_since(log_before).unwrap().len() as u64,
        report.written
    );
}

#[test]
fn a_dry_run_reports_without_writing() {
    let f = fixture();
    let log_before = f.store.max_sync_seq().unwrap();

    let report = f
        .store
        .backfill_session_at(&f.evidence, true, false)
        .unwrap();

    assert!(report.dry_run);
    assert_eq!(report.written, 0);
    assert_eq!(report.all.get(SessionTier::TranscriptLine), 3);
    assert_eq!(session_at(&f.store, &f.raw_one), None);
    assert_eq!(f.store.max_sync_seq().unwrap(), log_before);
}

#[test]
fn reemit_relogs_dates_that_are_already_filled() {
    let f = fixture();
    f.store
        .backfill_session_at(&f.evidence, false, false)
        .unwrap();
    let log_before = f.store.max_sync_seq().unwrap();

    let report = f
        .store
        .backfill_session_at(&f.evidence, false, true)
        .unwrap();

    assert_eq!(report.written, 0);
    assert_eq!(
        report.reemitted, 9,
        "every dated memory, the pre-dated one included"
    );
    assert_eq!(f.store.sync_log_since(log_before).unwrap().len(), 9);
}