second-brain-core 0.13.5

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::collections::HashMap;

use anyhow::Result;
use chrono::{DateTime, Utc};
use uuid::Uuid;

use crate::machine::MachineIdentity;
use crate::schema::{
    CompactStats, Conversation, Entity, IngestLogEntry, IngestRecord, Memory, MemoryType, Relation,
    RelationType, SyncEntry, SyncOp, SyncRecord, SyncState,
};

pub trait Store: Send + Sync {
    fn store_memory(&self, memory: &Memory) -> Result<()>;
    fn get_memory(&self, id: Uuid) -> Result<Option<Memory>>;
    fn get_memory_with_embedding(&self, id: Uuid) -> Result<Option<Memory>>;
    fn delete_memory(&self, id: Uuid) -> Result<()>;

    /// `delete_memory` that also reports whether a row was there to remove,
    /// so a forget of an unknown id can be answered as such instead of
    /// silently succeeding.
    fn remove_memory(&self, id: Uuid) -> Result<bool> {
        let existed = self.get_memory(id)?.is_some();
        self.delete_memory(id)?;
        Ok(existed)
    }

    fn store_entity(&self, entity: &Entity) -> Result<()>;
    fn get_entity(&self, id: Uuid) -> Result<Option<Entity>>;
    fn find_entity_by_name(&self, name: &str) -> Result<Option<Entity>>;

    fn store_conversation(&self, conversation: &Conversation) -> Result<()>;

    fn store_relation(&self, relation: &Relation) -> Result<()>;
    fn get_relations(
        &self,
        node_id: Uuid,
        relation_type: Option<RelationType>,
    ) -> Result<Vec<Relation>>;

    fn vector_search(&self, embedding: &[f32], limit: usize) -> Result<Vec<(Memory, f32)>>;

    /// Cosine search restricted to memories matching `source`, `memory_type`,
    /// and/or `project_path`, scanned exhaustively over that subset.
    ///
    /// A scoped query cannot be served by post-filtering an unfiltered
    /// shortlist, because the shortlist is chosen before the filter is known:
    /// a memory that does not place in the global top-k is then unreachable no
    /// matter how narrow the filter is. Scoping must constrain generation.
    fn filtered_vector_search(
        &self,
        embedding: &[f32],
        source: Option<&str>,
        memory_type: Option<MemoryType>,
        project_path: Option<&str>,
        limit: usize,
    ) -> Result<Vec<(Memory, f32)>>;

    fn traverse(&self, start_id: Uuid, depth: u32) -> Result<Vec<(Memory, Vec<Relation>)>>;

    fn memories_by_source(&self, source: &str) -> Result<Vec<Memory>>;
    fn memories_by_type(&self, memory_type: MemoryType) -> Result<Vec<Memory>>;

    /// Whether `memory` answers a recall scoped to `source`. Consolidation
    /// leaves the raws archived and the distillate tagged 'consolidation', so
    /// a store that tracks distillation must let a distillate answer for the
    /// sources of its raws, or a source filter finds nothing once its raws
    /// are consolidated. The default is the plain column comparison for
    /// stores that never consolidate.
    fn source_admits(&self, memory: &Memory, source: &str) -> Result<bool> {
        Ok(memory.source == source)
    }

    /// The text around a memory's chunk `ord`: that chunk joined with one
    /// neighbour on each side. None for a store that keeps no chunks.
    fn chunk_window(&self, memory_id: Uuid, ord: usize) -> Result<Option<String>> {
        let _ = (memory_id, ord);
        Ok(None)
    }

    fn update_memory(&self, memory: &Memory) -> Result<()>;

    fn record_access(&self, memory: &Memory) -> Result<()>;

    /// Count one access on each of `ids` and stamp them with the current
    /// time; confidence is left alone. A recall touches every result it
    /// returns, so an implementation should batch these into one write
    /// rather than take the writer lock per row. Ids the store does not
    /// hold are ignored.
    fn record_accesses(&self, ids: &[Uuid]) -> Result<()> {
        let now = Utc::now();
        for id in ids {
            if let Some(mut memory) = self.get_memory(*id)? {
                memory.access_count += 1;
                memory.last_accessed = now;
                self.record_access(&memory)?;
            }
        }
        Ok(())
    }

    fn bm25_search(&self, query: &str, limit: usize) -> Result<Vec<Memory>>;

    fn memory_count(&self) -> Result<usize>;

    fn all_memory_ids(&self) -> Result<Vec<Uuid>>;

    // Every Memory-rooted edge as a Relation, mirroring get_relations: strength
    // is always 1.0 and context is always None (get_relations discards edge
    // properties), so a bulk fetch is interchangeable with per-id reads. Used to
    // prewarm a relation cache in one scan instead of thousands of point reads.
    fn all_relations(&self) -> Result<Vec<Relation>>;
}

/// Live consolidation state, read from the memories table rather than the
/// append-only distillation_log, whose row count keeps counting raws that
/// were purged long ago.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ConsolidationStats {
    /// Hot, unsuperseded memories with source = 'consolidation'.
    pub distilled_live: u64,
    /// Hot, unsuperseded raws that no distillation pass has seen yet.
    pub raw_pending: u64,
    /// Distilled or skipped raws already moved to the archive tier.
    pub raw_retired: u64,
    /// Distilled or skipped raws still hot; the next retire pass moves them.
    pub raw_retire_pending: u64,
}

/// Raws waiting for a distillation pass, by the same rule as
/// `ConsolidationStats::raw_pending`.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct PendingDistillation {
    pub count: u64,
    pub oldest_created_at: Option<DateTime<Utc>>,
}

/// What `SqliteStore::restore_explicit_memories` changed.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ExplicitRestoreStats {
    /// Archived explicit raws moved back to the hot tier.
    pub raws_restored: u64,
    /// distillation_log rows removed from explicit raws.
    pub markers_dropped: u64,
    /// Distillates whose raws were all explicit, now superseded.
    pub distillates_superseded: u64,
}

/// What `SqliteStore::refile_stray_project_slugs` changed and what it
/// declined to decide.
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct StraySlugReport {
    pub memories_refiled: u64,
    pub refiled: Vec<RefiledSlug>,
    /// Names that end more than one stored project; left where they are.
    pub ambiguous: Vec<AmbiguousSlug>,
    /// Names that end no stored project; left where they are.
    pub unresolved: Vec<String>,
}

#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct RefiledSlug {
    pub from: String,
    pub to: String,
    pub memories: u64,
}

#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct AmbiguousSlug {
    pub slug: String,
    pub candidates: Vec<String>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ApplyOutcome {
    Created,
    Updated,
    Deleted,
    Skipped(SkipReason),
}

/// Why a sync record was not applied. Carried on the outcome because a bare
/// skip count cannot distinguish routine convergence (the row is already here)
/// from silent data loss (the edge can never resolve), and that ambiguity hid
/// tens of thousands of dropped relations for weeks.
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SkipReason {
    AlreadyPresent,
    Tombstoned,
    DanglingEndpoint,
    StaleWrite,
    Unsupported,
}

impl SkipReason {
    pub fn as_str(self) -> &'static str {
        match self {
            SkipReason::AlreadyPresent => "already_present",
            SkipReason::Tombstoned => "tombstoned",
            SkipReason::DanglingEndpoint => "dangling_endpoint",
            SkipReason::StaleWrite => "stale_write",
            SkipReason::Unsupported => "unsupported",
        }
    }
}

/// How many leading characters ingest compares to find a duplicate.
pub const CONTENT_PREFIX_CHARS: usize = 128;

/// The write/admin surface behind the daemon, sync, and maintenance CLI.
/// Split from Store so the query path stays lean enough for stub-store tests.
pub trait StoreAdmin: Store {
    fn register_machine(&self, identity: &MachineIdentity) -> Result<()>;
    fn get_all_machines(&self) -> Result<HashMap<String, String>>;
    fn diagnostic(&self) -> Result<String>;

    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
    /// Writes `memory` only when no memory holds its id in any state and the
    /// id is not tombstoned, checked in the insert's own transaction so two
    /// concurrent writers cannot both pass. True when it wrote. The default
    /// cannot check and always writes.
    fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
        self.store_memory(memory)?;
        Ok(true)
    }
    /// `store_memory_if_absent` for ingest, which may also refuse text a
    /// delete removed from the same source. The default refuses nothing more.
    fn ingest_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
        self.store_memory_if_absent(memory)
    }
    /// Whether ingest must skip `memory` because a delete removed its text
    /// from its source. A read outside any write, so it only spares work
    /// ahead of `ingest_memory_if_absent`. The default knows of no deletes.
    fn ingest_blocked(&self, _memory: &Memory) -> Result<bool> {
        Ok(false)
    }
    /// `store_memory_if_absent` for conversations: an existing row keeps its
    /// fields and nothing is logged for sync.
    fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
        self.store_conversation(conversation)?;
        Ok(true)
    }
    /// Whether a memory holds `id` in any state or the id is tombstoned. A
    /// read outside any write, so it can only spare work ahead of
    /// `store_memory_if_absent`, never replace it. The default knows nothing.
    fn memory_id_taken(&self, _id: Uuid) -> Result<bool> {
        Ok(false)
    }
    /// Whether the Claude Code session the transcript at `path` belongs to
    /// was deleted, so nothing of it may be captured again. The default
    /// knows of no deletes.
    fn session_deleted(&self, _path: &str) -> Result<bool> {
        Ok(false)
    }
    fn is_file_changed(&self, file_path: &str, file_hash: &str) -> Result<bool>;
    fn mark_ingested(
        &self,
        file_path: &str,
        file_hash: &str,
        memory_count: usize,
        source: &str,
    ) -> Result<()>;
    fn ingested_file_count(&self) -> Result<usize>;
    fn clear_ingest_log(&self) -> Result<usize>;
    fn ingest_log_entry(&self, _file_path: &str) -> Result<Option<IngestLogEntry>> {
        Ok(None)
    }
    /// Any row, at any path, recording a file with this session key and
    /// content hash.
    fn ingested_session(
        &self,
        _session_key: &str,
        _content_hash: &str,
    ) -> Result<Option<IngestLogEntry>> {
        Ok(None)
    }
    /// Legacy rows still holding an old-format hash whose file stem is
    /// exactly `stem`.
    fn legacy_ingest_entries_with_stem(&self, _stem: &str) -> Result<Vec<IngestLogEntry>> {
        Ok(Vec::new())
    }
    /// Replaces a row's hash with its SHA-256 and session key in place,
    /// keeping its legacy flag.
    fn upgrade_ingest_hash(
        &self,
        _file_path: &str,
        _content_hash: &str,
        _session_key: Option<&str>,
    ) -> Result<()> {
        Ok(())
    }
    fn record_ingested(&self, record: &IngestRecord<'_>) -> Result<()> {
        self.mark_ingested(
            record.file_path,
            record.content_hash,
            record.memory_count,
            record.source,
        )
    }
    /// Clears the rows a reingest may re-parse, keeping legacy rows: their
    /// raws deleted under random ids would come back under deterministic ones.
    fn clear_reparseable_ingest_log(&self) -> Result<usize> {
        self.clear_ingest_log()
    }
    fn delete_memories_by_source(&self, source: &str) -> Result<usize>;
    fn delete_memories_by_source_id(&self, source_id: &str) -> Result<usize>;
    fn rebuild_vector_index(&self) -> Result<()>;
    fn backfill_project_paths(&self) -> Result<u64>;
    fn backfill_sync_log(&self) -> Result<u64>;
    fn backfill_tombstones(&self) -> Result<u64>;
    fn prune_dangling_relations(&self) -> Result<u64>;
    /// Collapses re-logged duplicate `sync_log` rows down to one per
    /// `(node_type, node_id, op)`, keeping the highest `local_seq` in each
    /// group.
    fn compact_sync_log(&self, ops: &[SyncOp], dry_run: bool) -> Result<CompactStats>;
    fn memories_by_project_path(&self, project_path: &str) -> Result<Vec<Memory>>;
    /// Every distinct non-null `project_path` value with its live memory
    /// count, so a caller can discover the real slug `recall`'s `project`
    /// scope expects instead of guessing.
    fn distinct_project_paths(&self) -> Result<Vec<(String, u64)>>;
    fn all_memories_with_embeddings(&self) -> Result<Vec<Memory>>;

    fn consolidation_stats(&self) -> Result<ConsolidationStats>;
    /// Hot, unsuperseded raws with no distillation_log row, ordered by
    /// (source_id, created_at) so a batch reads as conversations.
    fn unconsolidated_memories(&self, limit: usize) -> Result<Vec<Memory>>;
    fn mark_consolidated(&self, raw_id: Uuid, distilled_id: Uuid, model: &str) -> Result<()>;
    /// Moves every distilled or skip-marked raw that is still hot to the
    /// archive tier. Nothing is deleted: the raw keeps its content and
    /// embedding and only leaves recall, so a distillate can always be
    /// traced back to what it was distilled from.
    fn retire_consolidated_raws(&self) -> Result<usize>;
    fn memories_created_between(
        &self,
        start: &DateTime<Utc>,
        end: &DateTime<Utc>,
    ) -> Result<Vec<Memory>>;

    fn sync_log_since(&self, after_seq: u64) -> Result<Vec<SyncEntry>>;
    fn sync_log_page(&self, after_seq: u64, limit: Option<usize>) -> Result<Vec<SyncEntry>>;
    /// A page to send a peer. A store that truncates its log refuses a page
    /// that starts below its floor.
    fn sync_log_export_page(&self, after_seq: u64, limit: Option<usize>) -> Result<Vec<SyncEntry>> {
        self.sync_log_page(after_seq, limit)
    }
    fn max_sync_seq(&self) -> Result<u64>;
    fn get_sync_state(&self, peer_id: &str) -> Result<Option<SyncState>>;
    fn set_sync_state(&self, state: &SyncState) -> Result<()>;
    fn get_all_sync_states(&self) -> Result<Vec<SyncState>>;
    fn tombstone_exists(&self, id: Uuid) -> Result<bool>;

    /// Applies one remote sync record with LWW/tombstone conflict semantics,
    /// in its own transaction.
    fn apply_record(&self, record: &SyncRecord) -> Result<ApplyOutcome>;

    /// Applies a batch atomically: either every record commits or none do, so
    /// a caller can fall back to per-record apply_record on error without
    /// double-applying a prefix.
    fn apply_batch(&self, records: &[SyncRecord]) -> Result<Vec<ApplyOutcome>>;
}