second-brain-core 0.13.5

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use uuid::Uuid;

fn epoch() -> DateTime<Utc> {
    DateTime::UNIX_EPOCH
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MemoryType {
    Episodic,
    Semantic,
    Procedural,
    Decision,
    Architecture,
    Debugging,
    Task,
    Question,
}

/// The source of a note an agent writes about its own session.
pub const AGENT_NOTE_SOURCE: &str = "agent-note";

/// Sources written through `remember` (over MCP, the HTTP API default,
/// `sb remember`, the ask service and its autonomous trigger), from
/// Claude's own memory files and markdown vaults, or as agent notes, rather
/// than mined from a transcript.
/// Consolidation must leave these alone: a paraphrase of an explicit note
/// loses exactly the wording its author chose.
pub const EXPLICIT_SOURCES: &[&str] = &[
    "mcp-direct",
    "api",
    "cli",
    "ask",
    "ask-auto",
    "claude-memory",
    AGENT_NOTE_SOURCE,
    "markdown",
];

/// Sources read from documents the user wrote in their repositories: commit
/// messages, pull request bodies and agent instruction files. Consolidation
/// leaves these alone too, because a newer version supersedes the raw it
/// replaces and a distillate of the old wording would outlive it.
pub const DOCUMENT_SOURCES: &[&str] = &["git-commit", "git-pr", "repo-instructions"];

/// SQL fragment over `$col` admitting only sources consolidation may
/// distill: neither distillates themselves nor the explicit or document
/// sources. The literals must list exactly `EXPLICIT_SOURCES` and
/// `DOCUMENT_SOURCES`; a test enforces it.
macro_rules! explicit_sources_list {
    () => {
        "'mcp-direct', 'api', 'cli', 'ask', 'ask-auto', 'claude-memory', 'agent-note', 'markdown'"
    };
}
macro_rules! document_sources_list {
    () => {
        "'git-commit', 'git-pr', 'repo-instructions'"
    };
}
macro_rules! distillable_source {
    ($col:literal) => {
        concat!(
            $col,
            " NOT IN ('consolidation', ",
            $crate::schema::explicit_sources_list!(),
            ", ",
            $crate::schema::document_sources_list!(),
            ")"
        )
    };
}
pub(crate) use {distillable_source, document_sources_list, explicit_sources_list};

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Memory {
    pub id: Uuid,
    pub content: String,
    pub embedding: Vec<f32>,
    pub memory_type: MemoryType,
    pub confidence: f32,
    pub created_at: DateTime<Utc>,
    pub last_accessed: DateTime<Utc>,
    pub access_count: u32,
    pub source: String,
    pub source_id: String,
    #[serde(default)]
    pub project_path: Option<String>,
    #[serde(default)]
    pub machine_id: String,
    #[serde(default = "epoch")]
    pub updated_at: DateTime<Utc>,
    /// LLM-generated chunk context for contextual retrieval; indexed and
    /// embedded together with `content`, backfilled nightly.
    #[serde(default)]
    pub context: Option<String>,
    #[serde(default)]
    pub conversation_id: Option<Uuid>,
    #[serde(default)]
    pub tier: Tier,
    /// Cold-tier segment id once archived; content is then a stub and the
    /// embedding is dropped from the row.
    #[serde(default)]
    pub archived_segment: Option<String>,
    /// Zep-style soft invalidation: a superseded memory stays in the store
    /// but leaves the retrieval indexes.
    #[serde(default)]
    pub superseded_by: Option<Uuid>,
    /// Set when `forget` retired the memory: it is archived like a
    /// consolidated raw but must never be restored with them.
    #[serde(default)]
    pub retired_reason: Option<String>,
    /// When the session this knowledge came from happened: the transcript
    /// line's time, or for a distillate the latest of its raws. None means
    /// no evidence survives, which must be shown as unknown, never guessed
    /// from `created_at` (that is the ingest or distill time).
    #[serde(default)]
    pub session_at: Option<DateTime<Utc>>,
    /// Who wrote the memory: the writer's user id, or when the writer named
    /// none this store's identity tag, `device:<machine id>` before it has
    /// one. None only on rows from before authors were recorded. Never
    /// changes once the row exists.
    #[serde(default)]
    pub author: Option<String>,
    #[serde(default)]
    pub scope: Scope,
}

/// Who may read a memory besides its author. Private memories reach only
/// their author; shared ones every user of the same store (a kiln tenant).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Scope {
    #[default]
    Private,
    Shared,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Tier {
    #[default]
    Hot,
    Archived,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Entity {
    pub id: Uuid,
    pub name: String,
    pub entity_type: String,
    pub embedding: Vec<f32>,
    pub aliases: Vec<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Conversation {
    pub id: Uuid,
    pub source: String,
    pub machine_id: String,
    pub started_at: DateTime<Utc>,
    pub project_path: Option<String>,
}

impl Conversation {
    /// Stable across stores and machines, so the same Claude Code session
    /// ingested twice lands on one conversation row, and a note filed under
    /// the session's id finds it.
    pub fn claude_code_id(session_key: &str) -> Uuid {
        Uuid::new_v5(
            &Uuid::NAMESPACE_OID,
            format!("claude-code:{session_key}").as_bytes(),
        )
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RelationType {
    RelatesTo,
    Mentions,
    DerivedFrom,
    Contradicts,
    Reinforces,
    Supersedes,
    DistilledFrom,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Relation {
    pub from_id: Uuid,
    pub to_id: Uuid,
    pub relation_type: RelationType,
    pub strength: f32,
    pub context: Option<String>,
}

impl Memory {
    pub fn new(
        content: String,
        memory_type: MemoryType,
        source: String,
        source_id: String,
    ) -> Self {
        let now = Utc::now();
        Self {
            id: Uuid::new_v4(),
            content,
            embedding: Vec::new(),
            memory_type,
            confidence: 1.0,
            created_at: now,
            last_accessed: now,
            access_count: 0,
            source,
            source_id,
            project_path: None,
            machine_id: String::new(),
            updated_at: now,
            context: None,
            conversation_id: None,
            tier: Tier::Hot,
            archived_segment: None,
            superseded_by: None,
            retired_reason: None,
            session_at: None,
            author: None,
            scope: Scope::Private,
        }
    }

    pub fn with_project_path(mut self, path: Option<String>) -> Self {
        self.project_path = path;
        self
    }

    pub fn with_machine_id(mut self, machine_id: String) -> Self {
        self.machine_id = machine_id;
        self
    }
}

impl Entity {
    pub fn new(name: String, entity_type: String) -> Self {
        Self {
            id: Uuid::new_v4(),
            name,
            entity_type,
            embedding: Vec::new(),
            aliases: Vec::new(),
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SyncOp {
    Create,
    Update,
    Delete,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SyncNodeType {
    Memory,
    Entity,
    Conversation,
    Relation,
    Distillation,
}

/// The record that a memory has already been distilled. Without this on the
/// wire only the distilled Memory replicates, never the fact that the source
/// was consumed, so every peer re-distills the same memories against the same
/// shared GPU.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Distillation {
    pub memory_id: Uuid,
    pub distilled_id: Option<Uuid>,
    pub distilled_at: Option<DateTime<Utc>>,
    pub model: Option<String>,
    pub status: String,
}

/// One `ingest_log` row. `content_hash` is None for a row written before
/// hashes became SHA-256, whose `file_hash` may then be the old 16-hex form.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct IngestLogEntry {
    pub file_path: String,
    pub file_hash: String,
    pub content_hash: Option<String>,
    pub session_key: Option<String>,
    /// Ingested before raw ids became deterministic.
    pub legacy_ids: bool,
}

/// What ingest records for a file it parsed or skipped. `legacy_ids` can
/// only set the flag on a row, never clear it.
#[derive(Debug, Clone, Copy)]
pub struct IngestRecord<'a> {
    pub file_path: &'a str,
    /// SHA-256 of the file, lowercase hex.
    pub content_hash: &'a str,
    pub session_key: Option<&'a str>,
    pub legacy_ids: bool,
    pub memory_count: usize,
    pub source: &'a str,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SyncEntry {
    pub id: String,
    pub local_seq: i64,
    pub origin_machine_id: String,
    pub origin_seq: i64,
    pub op: SyncOp,
    pub node_type: SyncNodeType,
    pub node_id: String,
    pub timestamp: DateTime<Utc>,
    pub data: Option<String>,
}

/// Wire form of a sync-log entry: same fields as SyncEntry but with `data`
/// parsed to JSON. Lives in core so StoreAdmin::apply_record can take it
/// without core depending on the sync crate.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SyncRecord {
    pub local_seq: u64,
    pub origin_machine_id: String,
    pub origin_seq: u64,
    pub op: SyncOp,
    pub node_type: SyncNodeType,
    pub node_id: String,
    pub timestamp: DateTime<Utc>,
    pub data: Option<serde_json::Value>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SyncState {
    pub peer_id: String,
    pub last_seq: u64,
    pub last_sync_at: DateTime<Utc>,
}

#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
pub struct CompactStats {
    pub groups_with_duplicates: u64,
    pub rows_deleted: u64,
    pub max_local_seq_before: u64,
    pub max_local_seq_after: u64,
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn agent_notes_are_explicit_so_consolidation_leaves_them_alone() {
        assert_eq!(AGENT_NOTE_SOURCE, "agent-note");
        assert!(EXPLICIT_SOURCES.contains(&AGENT_NOTE_SOURCE));
        assert!(distillable_source!("source").contains("'agent-note'"));
    }
}