Skip to main content

second_brain_core/
store.rs

1use std::collections::HashMap;
2
3use anyhow::Result;
4use chrono::{DateTime, Utc};
5use uuid::Uuid;
6
7use crate::machine::MachineIdentity;
8use crate::schema::{
9    CompactStats, Conversation, Entity, Memory, MemoryType, Relation, RelationType, SyncEntry,
10    SyncOp, SyncRecord, SyncState,
11};
12
13pub trait Store: Send + Sync {
14    fn store_memory(&self, memory: &Memory) -> Result<()>;
15    fn get_memory(&self, id: Uuid) -> Result<Option<Memory>>;
16    fn get_memory_with_embedding(&self, id: Uuid) -> Result<Option<Memory>>;
17    fn delete_memory(&self, id: Uuid) -> Result<()>;
18
19    /// `delete_memory` that also reports whether a row was there to remove,
20    /// so a forget of an unknown id can be answered as such instead of
21    /// silently succeeding.
22    fn remove_memory(&self, id: Uuid) -> Result<bool> {
23        let existed = self.get_memory(id)?.is_some();
24        self.delete_memory(id)?;
25        Ok(existed)
26    }
27
28    fn store_entity(&self, entity: &Entity) -> Result<()>;
29    fn get_entity(&self, id: Uuid) -> Result<Option<Entity>>;
30    fn find_entity_by_name(&self, name: &str) -> Result<Option<Entity>>;
31
32    fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
33
34    fn store_relation(&self, relation: &Relation) -> Result<()>;
35    fn get_relations(
36        &self,
37        node_id: Uuid,
38        relation_type: Option<RelationType>,
39    ) -> Result<Vec<Relation>>;
40
41    fn vector_search(&self, embedding: &[f32], limit: usize) -> Result<Vec<(Memory, f32)>>;
42
43    /// Cosine search restricted to memories matching `source`, `memory_type`,
44    /// and/or `project_path`, scanned exhaustively over that subset.
45    ///
46    /// A scoped query cannot be served by post-filtering an unfiltered
47    /// shortlist, because the shortlist is chosen before the filter is known:
48    /// a memory that does not place in the global top-k is then unreachable no
49    /// matter how narrow the filter is. Scoping must constrain generation.
50    fn filtered_vector_search(
51        &self,
52        embedding: &[f32],
53        source: Option<&str>,
54        memory_type: Option<MemoryType>,
55        project_path: Option<&str>,
56        limit: usize,
57    ) -> Result<Vec<(Memory, f32)>>;
58
59    fn traverse(&self, start_id: Uuid, depth: u32) -> Result<Vec<(Memory, Vec<Relation>)>>;
60
61    fn memories_by_source(&self, source: &str) -> Result<Vec<Memory>>;
62    fn memories_by_type(&self, memory_type: MemoryType) -> Result<Vec<Memory>>;
63
64    /// Whether `memory` answers a recall scoped to `source`. Consolidation
65    /// leaves the raws archived and the distillate tagged 'consolidation', so
66    /// a store that tracks distillation must let a distillate answer for the
67    /// sources of its raws, or a source filter finds nothing once its raws
68    /// are consolidated. The default is the plain column comparison for
69    /// stores that never consolidate.
70    fn source_admits(&self, memory: &Memory, source: &str) -> Result<bool> {
71        Ok(memory.source == source)
72    }
73
74    fn update_memory(&self, memory: &Memory) -> Result<()>;
75
76    fn record_access(&self, memory: &Memory) -> Result<()>;
77
78    /// Count one access on each of `ids` and stamp them with the current
79    /// time; confidence is left alone. A recall touches every result it
80    /// returns, so an implementation should batch these into one write
81    /// rather than take the writer lock per row. Ids the store does not
82    /// hold are ignored.
83    fn record_accesses(&self, ids: &[Uuid]) -> Result<()> {
84        let now = Utc::now();
85        for id in ids {
86            if let Some(mut memory) = self.get_memory(*id)? {
87                memory.access_count += 1;
88                memory.last_accessed = now;
89                self.record_access(&memory)?;
90            }
91        }
92        Ok(())
93    }
94
95    fn bm25_search(&self, query: &str, limit: usize) -> Result<Vec<Memory>>;
96
97    fn memory_count(&self) -> Result<usize>;
98
99    fn all_memory_ids(&self) -> Result<Vec<Uuid>>;
100
101    // Every Memory-rooted edge as a Relation, mirroring get_relations: strength
102    // is always 1.0 and context is always None (get_relations discards edge
103    // properties), so a bulk fetch is interchangeable with per-id reads. Used to
104    // prewarm a relation cache in one scan instead of thousands of point reads.
105    fn all_relations(&self) -> Result<Vec<Relation>>;
106}
107
108/// Live consolidation state, read from the memories table rather than the
109/// append-only distillation_log, whose row count keeps counting raws that
110/// were purged long ago.
111#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
112pub struct ConsolidationStats {
113    /// Hot, unsuperseded memories with source = 'consolidation'.
114    pub distilled_live: u64,
115    /// Hot, unsuperseded raws that no distillation pass has seen yet.
116    pub raw_pending: u64,
117    /// Distilled or skipped raws already moved to the archive tier.
118    pub raw_retired: u64,
119    /// Distilled or skipped raws still hot; the next retire pass moves them.
120    pub raw_retire_pending: u64,
121}
122
123/// What `SqliteStore::restore_explicit_memories` changed.
124#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
125pub struct ExplicitRestoreStats {
126    /// Archived explicit raws moved back to the hot tier.
127    pub raws_restored: u64,
128    /// distillation_log rows removed from explicit raws.
129    pub markers_dropped: u64,
130    /// Distillates whose raws were all explicit, now superseded.
131    pub distillates_superseded: u64,
132}
133
134/// What `SqliteStore::refile_stray_project_slugs` changed and what it
135/// declined to decide.
136#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
137pub struct StraySlugReport {
138    pub memories_refiled: u64,
139    pub refiled: Vec<RefiledSlug>,
140    /// Names that end more than one stored project; left where they are.
141    pub ambiguous: Vec<AmbiguousSlug>,
142    /// Names that end no stored project; left where they are.
143    pub unresolved: Vec<String>,
144}
145
146#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
147pub struct RefiledSlug {
148    pub from: String,
149    pub to: String,
150    pub memories: u64,
151}
152
153#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
154pub struct AmbiguousSlug {
155    pub slug: String,
156    pub candidates: Vec<String>,
157}
158
159#[derive(Debug, Clone, Copy, PartialEq, Eq)]
160pub enum ApplyOutcome {
161    Created,
162    Updated,
163    Deleted,
164    Skipped(SkipReason),
165}
166
167/// Why a sync record was not applied. Carried on the outcome because a bare
168/// skip count cannot distinguish routine convergence (the row is already here)
169/// from silent data loss (the edge can never resolve), and that ambiguity hid
170/// tens of thousands of dropped relations for weeks.
171#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
172#[serde(rename_all = "snake_case")]
173pub enum SkipReason {
174    AlreadyPresent,
175    Tombstoned,
176    DanglingEndpoint,
177    StaleWrite,
178    Unsupported,
179}
180
181impl SkipReason {
182    pub fn as_str(self) -> &'static str {
183        match self {
184            SkipReason::AlreadyPresent => "already_present",
185            SkipReason::Tombstoned => "tombstoned",
186            SkipReason::DanglingEndpoint => "dangling_endpoint",
187            SkipReason::StaleWrite => "stale_write",
188            SkipReason::Unsupported => "unsupported",
189        }
190    }
191}
192
193/// The write/admin surface behind the daemon, sync, and maintenance CLI.
194/// Split from Store so the query path stays lean enough for stub-store tests.
195pub trait StoreAdmin: Store {
196    fn register_machine(&self, identity: &MachineIdentity) -> Result<()>;
197    fn get_all_machines(&self) -> Result<HashMap<String, String>>;
198    fn diagnostic(&self) -> Result<String>;
199
200    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
201    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
202    fn is_file_changed(&self, file_path: &str, file_hash: &str) -> Result<bool>;
203    fn mark_ingested(
204        &self,
205        file_path: &str,
206        file_hash: &str,
207        memory_count: usize,
208        source: &str,
209    ) -> Result<()>;
210    fn ingested_file_count(&self) -> Result<usize>;
211    fn clear_ingest_log(&self) -> Result<usize>;
212    fn delete_memories_by_source(&self, source: &str) -> Result<usize>;
213    fn delete_memories_by_source_id(&self, source_id: &str) -> Result<usize>;
214    fn rebuild_vector_index(&self) -> Result<()>;
215    fn backfill_project_paths(&self) -> Result<u64>;
216    fn backfill_sync_log(&self) -> Result<u64>;
217    fn backfill_tombstones(&self) -> Result<u64>;
218    fn prune_dangling_relations(&self) -> Result<u64>;
219    /// Collapses re-logged duplicate `sync_log` rows down to one per
220    /// `(node_type, node_id, op)`, keeping the highest `local_seq` in each
221    /// group since the counter is derived as `MAX(local_seq)+1` on every
222    /// write: dropping the row holding the true max would let the next
223    /// write reuse a sequence a peer already considers seen.
224    fn compact_sync_log(&self, ops: &[SyncOp], dry_run: bool) -> Result<CompactStats>;
225    fn memories_by_project_path(&self, project_path: &str) -> Result<Vec<Memory>>;
226    /// Every distinct non-null `project_path` value with its live memory
227    /// count, so a caller can discover the real slug `recall`'s `project`
228    /// scope expects instead of guessing.
229    fn distinct_project_paths(&self) -> Result<Vec<(String, u64)>>;
230    fn all_memories_with_embeddings(&self) -> Result<Vec<Memory>>;
231
232    fn consolidation_stats(&self) -> Result<ConsolidationStats>;
233    /// Hot, unsuperseded raws with no distillation_log row, ordered by
234    /// (source_id, created_at) so a batch reads as conversations.
235    fn unconsolidated_memories(&self, limit: usize) -> Result<Vec<Memory>>;
236    fn mark_consolidated(&self, raw_id: Uuid, distilled_id: Uuid, model: &str) -> Result<()>;
237    /// Moves every distilled or skip-marked raw that is still hot to the
238    /// archive tier. Nothing is deleted: the raw keeps its content and
239    /// embedding and only leaves recall, so a distillate can always be
240    /// traced back to what it was distilled from.
241    fn retire_consolidated_raws(&self) -> Result<usize>;
242    fn memories_created_between(
243        &self,
244        start: &DateTime<Utc>,
245        end: &DateTime<Utc>,
246    ) -> Result<Vec<Memory>>;
247
248    fn sync_log_since(&self, after_seq: u64) -> Result<Vec<SyncEntry>>;
249    fn sync_log_page(&self, after_seq: u64, limit: Option<usize>) -> Result<Vec<SyncEntry>>;
250    fn max_sync_seq(&self) -> Result<u64>;
251    fn get_sync_state(&self, peer_id: &str) -> Result<Option<SyncState>>;
252    fn set_sync_state(&self, state: &SyncState) -> Result<()>;
253    fn get_all_sync_states(&self) -> Result<Vec<SyncState>>;
254    fn tombstone_exists(&self, id: Uuid) -> Result<bool>;
255
256    /// Applies one remote sync record with LWW/tombstone conflict semantics,
257    /// in its own transaction.
258    fn apply_record(&self, record: &SyncRecord) -> Result<ApplyOutcome>;
259
260    /// Applies a batch atomically: either every record commits or none do, so
261    /// a caller can fall back to per-record apply_record on error without
262    /// double-applying a prefix.
263    fn apply_batch(&self, records: &[SyncRecord]) -> Result<Vec<ApplyOutcome>>;
264}