Skip to main content

second_brain_core/
store.rs

1use std::collections::HashMap;
2
3use anyhow::Result;
4use chrono::{DateTime, Utc};
5use uuid::Uuid;
6
7use crate::machine::MachineIdentity;
8use crate::schema::{
9    CompactStats, Conversation, Entity, Memory, MemoryType, Relation, RelationType, SyncEntry,
10    SyncOp, SyncRecord, SyncState,
11};
12
13pub trait Store: Send + Sync {
14    fn store_memory(&self, memory: &Memory) -> Result<()>;
15    fn get_memory(&self, id: Uuid) -> Result<Option<Memory>>;
16    fn get_memory_with_embedding(&self, id: Uuid) -> Result<Option<Memory>>;
17    fn delete_memory(&self, id: Uuid) -> Result<()>;
18
19    /// `delete_memory` that also reports whether a row was there to remove,
20    /// so a forget of an unknown id can be answered as such instead of
21    /// silently succeeding.
22    fn remove_memory(&self, id: Uuid) -> Result<bool> {
23        let existed = self.get_memory(id)?.is_some();
24        self.delete_memory(id)?;
25        Ok(existed)
26    }
27
28    fn store_entity(&self, entity: &Entity) -> Result<()>;
29    fn get_entity(&self, id: Uuid) -> Result<Option<Entity>>;
30    fn find_entity_by_name(&self, name: &str) -> Result<Option<Entity>>;
31
32    fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
33
34    fn store_relation(&self, relation: &Relation) -> Result<()>;
35    fn get_relations(
36        &self,
37        node_id: Uuid,
38        relation_type: Option<RelationType>,
39    ) -> Result<Vec<Relation>>;
40
41    fn vector_search(&self, embedding: &[f32], limit: usize) -> Result<Vec<(Memory, f32)>>;
42
43    /// Cosine search restricted to memories matching `source`, `memory_type`,
44    /// and/or `project_path`, scanned exhaustively over that subset.
45    ///
46    /// A scoped query cannot be served by post-filtering an unfiltered
47    /// shortlist, because the shortlist is chosen before the filter is known:
48    /// a memory that does not place in the global top-k is then unreachable no
49    /// matter how narrow the filter is. Scoping must constrain generation.
50    fn filtered_vector_search(
51        &self,
52        embedding: &[f32],
53        source: Option<&str>,
54        memory_type: Option<MemoryType>,
55        project_path: Option<&str>,
56        limit: usize,
57    ) -> Result<Vec<(Memory, f32)>>;
58
59    fn traverse(&self, start_id: Uuid, depth: u32) -> Result<Vec<(Memory, Vec<Relation>)>>;
60
61    fn memories_by_source(&self, source: &str) -> Result<Vec<Memory>>;
62    fn memories_by_type(&self, memory_type: MemoryType) -> Result<Vec<Memory>>;
63
64    /// Whether `memory` answers a recall scoped to `source`. Consolidation
65    /// leaves the raws archived and the distillate tagged 'consolidation', so
66    /// a store that tracks distillation must let a distillate answer for the
67    /// sources of its raws, or a source filter finds nothing once its raws
68    /// are consolidated. The default is the plain column comparison for
69    /// stores that never consolidate.
70    fn source_admits(&self, memory: &Memory, source: &str) -> Result<bool> {
71        Ok(memory.source == source)
72    }
73
74    /// The text around a memory's chunk `ord`: that chunk joined with one
75    /// neighbour on each side. None for a store that keeps no chunks.
76    fn chunk_window(&self, memory_id: Uuid, ord: usize) -> Result<Option<String>> {
77        let _ = (memory_id, ord);
78        Ok(None)
79    }
80
81    fn update_memory(&self, memory: &Memory) -> Result<()>;
82
83    fn record_access(&self, memory: &Memory) -> Result<()>;
84
85    /// Count one access on each of `ids` and stamp them with the current
86    /// time; confidence is left alone. A recall touches every result it
87    /// returns, so an implementation should batch these into one write
88    /// rather than take the writer lock per row. Ids the store does not
89    /// hold are ignored.
90    fn record_accesses(&self, ids: &[Uuid]) -> Result<()> {
91        let now = Utc::now();
92        for id in ids {
93            if let Some(mut memory) = self.get_memory(*id)? {
94                memory.access_count += 1;
95                memory.last_accessed = now;
96                self.record_access(&memory)?;
97            }
98        }
99        Ok(())
100    }
101
102    fn bm25_search(&self, query: &str, limit: usize) -> Result<Vec<Memory>>;
103
104    fn memory_count(&self) -> Result<usize>;
105
106    fn all_memory_ids(&self) -> Result<Vec<Uuid>>;
107
108    // Every Memory-rooted edge as a Relation, mirroring get_relations: strength
109    // is always 1.0 and context is always None (get_relations discards edge
110    // properties), so a bulk fetch is interchangeable with per-id reads. Used to
111    // prewarm a relation cache in one scan instead of thousands of point reads.
112    fn all_relations(&self) -> Result<Vec<Relation>>;
113}
114
115/// Live consolidation state, read from the memories table rather than the
116/// append-only distillation_log, whose row count keeps counting raws that
117/// were purged long ago.
118#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
119pub struct ConsolidationStats {
120    /// Hot, unsuperseded memories with source = 'consolidation'.
121    pub distilled_live: u64,
122    /// Hot, unsuperseded raws that no distillation pass has seen yet.
123    pub raw_pending: u64,
124    /// Distilled or skipped raws already moved to the archive tier.
125    pub raw_retired: u64,
126    /// Distilled or skipped raws still hot; the next retire pass moves them.
127    pub raw_retire_pending: u64,
128}
129
130/// What `SqliteStore::restore_explicit_memories` changed.
131#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
132pub struct ExplicitRestoreStats {
133    /// Archived explicit raws moved back to the hot tier.
134    pub raws_restored: u64,
135    /// distillation_log rows removed from explicit raws.
136    pub markers_dropped: u64,
137    /// Distillates whose raws were all explicit, now superseded.
138    pub distillates_superseded: u64,
139}
140
141/// What `SqliteStore::refile_stray_project_slugs` changed and what it
142/// declined to decide.
143#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
144pub struct StraySlugReport {
145    pub memories_refiled: u64,
146    pub refiled: Vec<RefiledSlug>,
147    /// Names that end more than one stored project; left where they are.
148    pub ambiguous: Vec<AmbiguousSlug>,
149    /// Names that end no stored project; left where they are.
150    pub unresolved: Vec<String>,
151}
152
153#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
154pub struct RefiledSlug {
155    pub from: String,
156    pub to: String,
157    pub memories: u64,
158}
159
160#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
161pub struct AmbiguousSlug {
162    pub slug: String,
163    pub candidates: Vec<String>,
164}
165
166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
167pub enum ApplyOutcome {
168    Created,
169    Updated,
170    Deleted,
171    Skipped(SkipReason),
172}
173
174/// Why a sync record was not applied. Carried on the outcome because a bare
175/// skip count cannot distinguish routine convergence (the row is already here)
176/// from silent data loss (the edge can never resolve), and that ambiguity hid
177/// tens of thousands of dropped relations for weeks.
178#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
179#[serde(rename_all = "snake_case")]
180pub enum SkipReason {
181    AlreadyPresent,
182    Tombstoned,
183    DanglingEndpoint,
184    StaleWrite,
185    Unsupported,
186}
187
188impl SkipReason {
189    pub fn as_str(self) -> &'static str {
190        match self {
191            SkipReason::AlreadyPresent => "already_present",
192            SkipReason::Tombstoned => "tombstoned",
193            SkipReason::DanglingEndpoint => "dangling_endpoint",
194            SkipReason::StaleWrite => "stale_write",
195            SkipReason::Unsupported => "unsupported",
196        }
197    }
198}
199
200/// The write/admin surface behind the daemon, sync, and maintenance CLI.
201/// Split from Store so the query path stays lean enough for stub-store tests.
202pub trait StoreAdmin: Store {
203    fn register_machine(&self, identity: &MachineIdentity) -> Result<()>;
204    fn get_all_machines(&self) -> Result<HashMap<String, String>>;
205    fn diagnostic(&self) -> Result<String>;
206
207    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
208    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
209    fn is_file_changed(&self, file_path: &str, file_hash: &str) -> Result<bool>;
210    fn mark_ingested(
211        &self,
212        file_path: &str,
213        file_hash: &str,
214        memory_count: usize,
215        source: &str,
216    ) -> Result<()>;
217    fn ingested_file_count(&self) -> Result<usize>;
218    fn clear_ingest_log(&self) -> Result<usize>;
219    fn delete_memories_by_source(&self, source: &str) -> Result<usize>;
220    fn delete_memories_by_source_id(&self, source_id: &str) -> Result<usize>;
221    fn rebuild_vector_index(&self) -> Result<()>;
222    fn backfill_project_paths(&self) -> Result<u64>;
223    fn backfill_sync_log(&self) -> Result<u64>;
224    fn backfill_tombstones(&self) -> Result<u64>;
225    fn prune_dangling_relations(&self) -> Result<u64>;
226    /// Collapses re-logged duplicate `sync_log` rows down to one per
227    /// `(node_type, node_id, op)`, keeping the highest `local_seq` in each
228    /// group since the counter is derived as `MAX(local_seq)+1` on every
229    /// write: dropping the row holding the true max would let the next
230    /// write reuse a sequence a peer already considers seen.
231    fn compact_sync_log(&self, ops: &[SyncOp], dry_run: bool) -> Result<CompactStats>;
232    fn memories_by_project_path(&self, project_path: &str) -> Result<Vec<Memory>>;
233    /// Every distinct non-null `project_path` value with its live memory
234    /// count, so a caller can discover the real slug `recall`'s `project`
235    /// scope expects instead of guessing.
236    fn distinct_project_paths(&self) -> Result<Vec<(String, u64)>>;
237    fn all_memories_with_embeddings(&self) -> Result<Vec<Memory>>;
238
239    fn consolidation_stats(&self) -> Result<ConsolidationStats>;
240    /// Hot, unsuperseded raws with no distillation_log row, ordered by
241    /// (source_id, created_at) so a batch reads as conversations.
242    fn unconsolidated_memories(&self, limit: usize) -> Result<Vec<Memory>>;
243    fn mark_consolidated(&self, raw_id: Uuid, distilled_id: Uuid, model: &str) -> Result<()>;
244    /// Moves every distilled or skip-marked raw that is still hot to the
245    /// archive tier. Nothing is deleted: the raw keeps its content and
246    /// embedding and only leaves recall, so a distillate can always be
247    /// traced back to what it was distilled from.
248    fn retire_consolidated_raws(&self) -> Result<usize>;
249    fn memories_created_between(
250        &self,
251        start: &DateTime<Utc>,
252        end: &DateTime<Utc>,
253    ) -> Result<Vec<Memory>>;
254
255    fn sync_log_since(&self, after_seq: u64) -> Result<Vec<SyncEntry>>;
256    fn sync_log_page(&self, after_seq: u64, limit: Option<usize>) -> Result<Vec<SyncEntry>>;
257    fn max_sync_seq(&self) -> Result<u64>;
258    fn get_sync_state(&self, peer_id: &str) -> Result<Option<SyncState>>;
259    fn set_sync_state(&self, state: &SyncState) -> Result<()>;
260    fn get_all_sync_states(&self) -> Result<Vec<SyncState>>;
261    fn tombstone_exists(&self, id: Uuid) -> Result<bool>;
262
263    /// Applies one remote sync record with LWW/tombstone conflict semantics,
264    /// in its own transaction.
265    fn apply_record(&self, record: &SyncRecord) -> Result<ApplyOutcome>;
266
267    /// Applies a batch atomically: either every record commits or none do, so
268    /// a caller can fall back to per-record apply_record on error without
269    /// double-applying a prefix.
270    fn apply_batch(&self, records: &[SyncRecord]) -> Result<Vec<ApplyOutcome>>;
271}