second-brain-core 0.9.0

Core library for second-brain: KuzuDB graph storage, BGE embeddings, and weighted query engine
Documentation
use std::collections::HashMap;

use anyhow::Result;
use chrono::{DateTime, Utc};
use uuid::Uuid;

use crate::machine::MachineIdentity;
use crate::schema::{
    CompactStats, Conversation, Entity, Memory, MemoryType, Relation, RelationType, SyncEntry,
    SyncOp, SyncRecord, SyncState,
};

pub trait Store: Send + Sync {
    fn store_memory(&self, memory: &Memory) -> Result<()>;
    fn get_memory(&self, id: Uuid) -> Result<Option<Memory>>;
    fn get_memory_with_embedding(&self, id: Uuid) -> Result<Option<Memory>>;
    fn delete_memory(&self, id: Uuid) -> Result<()>;

    /// `delete_memory` that also reports whether a row was there to remove,
    /// so a forget of an unknown id can be answered as such instead of
    /// silently succeeding.
    fn remove_memory(&self, id: Uuid) -> Result<bool> {
        let existed = self.get_memory(id)?.is_some();
        self.delete_memory(id)?;
        Ok(existed)
    }

    fn store_entity(&self, entity: &Entity) -> Result<()>;
    fn get_entity(&self, id: Uuid) -> Result<Option<Entity>>;
    fn find_entity_by_name(&self, name: &str) -> Result<Option<Entity>>;

    fn store_conversation(&self, conversation: &Conversation) -> Result<()>;

    fn store_relation(&self, relation: &Relation) -> Result<()>;
    fn get_relations(
        &self,
        node_id: Uuid,
        relation_type: Option<RelationType>,
    ) -> Result<Vec<Relation>>;

    fn vector_search(&self, embedding: &[f32], limit: usize) -> Result<Vec<(Memory, f32)>>;

    /// Cosine search restricted to memories matching `source`, `memory_type`,
    /// and/or `project_path`, scanned exhaustively over that subset.
    ///
    /// A scoped query cannot be served by post-filtering an unfiltered
    /// shortlist, because the shortlist is chosen before the filter is known:
    /// a memory that does not place in the global top-k is then unreachable no
    /// matter how narrow the filter is. Scoping must constrain generation.
    fn filtered_vector_search(
        &self,
        embedding: &[f32],
        source: Option<&str>,
        memory_type: Option<MemoryType>,
        project_path: Option<&str>,
        limit: usize,
    ) -> Result<Vec<(Memory, f32)>>;

    fn traverse(&self, start_id: Uuid, depth: u32) -> Result<Vec<(Memory, Vec<Relation>)>>;

    fn memories_by_source(&self, source: &str) -> Result<Vec<Memory>>;
    fn memories_by_type(&self, memory_type: MemoryType) -> Result<Vec<Memory>>;

    /// Whether `memory` answers a recall scoped to `source`. Consolidation
    /// leaves the raws archived and the distillate tagged 'consolidation', so
    /// a store that tracks distillation must let a distillate answer for the
    /// sources of its raws, or a source filter finds nothing once its raws
    /// are consolidated. The default is the plain column comparison for
    /// stores that never consolidate.
    fn source_admits(&self, memory: &Memory, source: &str) -> Result<bool> {
        Ok(memory.source == source)
    }

    fn update_memory(&self, memory: &Memory) -> Result<()>;

    fn record_access(&self, memory: &Memory) -> Result<()>;

    /// Count one access on each of `ids` and stamp them with the current
    /// time; confidence is left alone. A recall touches every result it
    /// returns, so an implementation should batch these into one write
    /// rather than take the writer lock per row. Ids the store does not
    /// hold are ignored.
    fn record_accesses(&self, ids: &[Uuid]) -> Result<()> {
        let now = Utc::now();
        for id in ids {
            if let Some(mut memory) = self.get_memory(*id)? {
                memory.access_count += 1;
                memory.last_accessed = now;
                self.record_access(&memory)?;
            }
        }
        Ok(())
    }

    fn bm25_search(&self, query: &str, limit: usize) -> Result<Vec<Memory>>;

    fn memory_count(&self) -> Result<usize>;

    fn all_memory_ids(&self) -> Result<Vec<Uuid>>;

    // Every Memory-rooted edge as a Relation, mirroring get_relations: strength
    // is always 1.0 and context is always None (get_relations discards edge
    // properties), so a bulk fetch is interchangeable with per-id reads. Used to
    // prewarm a relation cache in one scan instead of thousands of point reads.
    fn all_relations(&self) -> Result<Vec<Relation>>;
}

/// Live consolidation state, read from the memories table rather than the
/// append-only distillation_log, whose row count keeps counting raws that
/// were purged long ago.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ConsolidationStats {
    /// Hot, unsuperseded memories with source = 'consolidation'.
    pub distilled_live: u64,
    /// Hot, unsuperseded raws that no distillation pass has seen yet.
    pub raw_pending: u64,
    /// Distilled or skipped raws already moved to the archive tier.
    pub raw_retired: u64,
    /// Distilled or skipped raws still hot; the next retire pass moves them.
    pub raw_retire_pending: u64,
}

/// What `SqliteStore::restore_explicit_memories` changed.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ExplicitRestoreStats {
    /// Archived explicit raws moved back to the hot tier.
    pub raws_restored: u64,
    /// distillation_log rows removed from explicit raws.
    pub markers_dropped: u64,
    /// Distillates whose raws were all explicit, now superseded.
    pub distillates_superseded: u64,
}

/// What `SqliteStore::refile_stray_project_slugs` changed and what it
/// declined to decide.
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct StraySlugReport {
    pub memories_refiled: u64,
    pub refiled: Vec<RefiledSlug>,
    /// Names that end more than one stored project; left where they are.
    pub ambiguous: Vec<AmbiguousSlug>,
    /// Names that end no stored project; left where they are.
    pub unresolved: Vec<String>,
}

#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct RefiledSlug {
    pub from: String,
    pub to: String,
    pub memories: u64,
}

#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct AmbiguousSlug {
    pub slug: String,
    pub candidates: Vec<String>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ApplyOutcome {
    Created,
    Updated,
    Deleted,
    Skipped(SkipReason),
}

/// Why a sync record was not applied. Carried on the outcome because a bare
/// skip count cannot distinguish routine convergence (the row is already here)
/// from silent data loss (the edge can never resolve), and that ambiguity hid
/// tens of thousands of dropped relations for weeks.
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SkipReason {
    AlreadyPresent,
    Tombstoned,
    DanglingEndpoint,
    StaleWrite,
    Unsupported,
}

impl SkipReason {
    pub fn as_str(self) -> &'static str {
        match self {
            SkipReason::AlreadyPresent => "already_present",
            SkipReason::Tombstoned => "tombstoned",
            SkipReason::DanglingEndpoint => "dangling_endpoint",
            SkipReason::StaleWrite => "stale_write",
            SkipReason::Unsupported => "unsupported",
        }
    }
}

/// The write/admin surface behind the daemon, sync, and maintenance CLI.
/// Split from Store so the query path stays lean enough for stub-store tests.
pub trait StoreAdmin: Store {
    fn register_machine(&self, identity: &MachineIdentity) -> Result<()>;
    fn get_all_machines(&self) -> Result<HashMap<String, String>>;
    fn diagnostic(&self) -> Result<String>;

    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
    fn is_file_changed(&self, file_path: &str, file_hash: &str) -> Result<bool>;
    fn mark_ingested(
        &self,
        file_path: &str,
        file_hash: &str,
        memory_count: usize,
        source: &str,
    ) -> Result<()>;
    fn ingested_file_count(&self) -> Result<usize>;
    fn clear_ingest_log(&self) -> Result<usize>;
    fn delete_memories_by_source(&self, source: &str) -> Result<usize>;
    fn delete_memories_by_source_id(&self, source_id: &str) -> Result<usize>;
    fn rebuild_vector_index(&self) -> Result<()>;
    fn backfill_project_paths(&self) -> Result<u64>;
    fn backfill_sync_log(&self) -> Result<u64>;
    fn backfill_tombstones(&self) -> Result<u64>;
    fn prune_dangling_relations(&self) -> Result<u64>;
    /// Collapses re-logged duplicate `sync_log` rows down to one per
    /// `(node_type, node_id, op)`, keeping the highest `local_seq` in each
    /// group since the counter is derived as `MAX(local_seq)+1` on every
    /// write: dropping the row holding the true max would let the next
    /// write reuse a sequence a peer already considers seen.
    fn compact_sync_log(&self, ops: &[SyncOp], dry_run: bool) -> Result<CompactStats>;
    fn memories_by_project_path(&self, project_path: &str) -> Result<Vec<Memory>>;
    /// Every distinct non-null `project_path` value with its live memory
    /// count, so a caller can discover the real slug `recall`'s `project`
    /// scope expects instead of guessing.
    fn distinct_project_paths(&self) -> Result<Vec<(String, u64)>>;
    fn all_memories_with_embeddings(&self) -> Result<Vec<Memory>>;

    fn consolidation_stats(&self) -> Result<ConsolidationStats>;
    /// Hot, unsuperseded raws with no distillation_log row, ordered by
    /// (source_id, created_at) so a batch reads as conversations.
    fn unconsolidated_memories(&self, limit: usize) -> Result<Vec<Memory>>;
    fn mark_consolidated(&self, raw_id: Uuid, distilled_id: Uuid, model: &str) -> Result<()>;
    /// Moves every distilled or skip-marked raw that is still hot to the
    /// archive tier. Nothing is deleted: the raw keeps its content and
    /// embedding and only leaves recall, so a distillate can always be
    /// traced back to what it was distilled from.
    fn retire_consolidated_raws(&self) -> Result<usize>;
    fn memories_created_between(
        &self,
        start: &DateTime<Utc>,
        end: &DateTime<Utc>,
    ) -> Result<Vec<Memory>>;

    fn sync_log_since(&self, after_seq: u64) -> Result<Vec<SyncEntry>>;
    fn sync_log_page(&self, after_seq: u64, limit: Option<usize>) -> Result<Vec<SyncEntry>>;
    fn max_sync_seq(&self) -> Result<u64>;
    fn get_sync_state(&self, peer_id: &str) -> Result<Option<SyncState>>;
    fn set_sync_state(&self, state: &SyncState) -> Result<()>;
    fn get_all_sync_states(&self) -> Result<Vec<SyncState>>;
    fn tombstone_exists(&self, id: Uuid) -> Result<bool>;

    /// Applies one remote sync record with LWW/tombstone conflict semantics,
    /// in its own transaction.
    fn apply_record(&self, record: &SyncRecord) -> Result<ApplyOutcome>;

    /// Applies a batch atomically: either every record commits or none do, so
    /// a caller can fall back to per-record apply_record on error without
    /// double-applying a prefix.
    fn apply_batch(&self, records: &[SyncRecord]) -> Result<Vec<ApplyOutcome>>;
}