Skip to main content

second_brain_core/
store.rs

1use std::collections::HashMap;
2
3use anyhow::Result;
4use chrono::{DateTime, Utc};
5use uuid::Uuid;
6
7use crate::machine::MachineIdentity;
8use crate::schema::{
9    CompactStats, Conversation, Entity, IngestLogEntry, IngestRecord, Memory, MemoryType, Relation,
10    RelationType, SyncEntry, SyncOp, SyncRecord, SyncState,
11};
12
13pub trait Store: Send + Sync {
14    fn store_memory(&self, memory: &Memory) -> Result<()>;
15    fn get_memory(&self, id: Uuid) -> Result<Option<Memory>>;
16    fn get_memory_with_embedding(&self, id: Uuid) -> Result<Option<Memory>>;
17    fn delete_memory(&self, id: Uuid) -> Result<()>;
18
19    /// `delete_memory` that also reports whether a row was there to remove,
20    /// so a forget of an unknown id can be answered as such instead of
21    /// silently succeeding.
22    fn remove_memory(&self, id: Uuid) -> Result<bool> {
23        let existed = self.get_memory(id)?.is_some();
24        self.delete_memory(id)?;
25        Ok(existed)
26    }
27
28    fn store_entity(&self, entity: &Entity) -> Result<()>;
29    fn get_entity(&self, id: Uuid) -> Result<Option<Entity>>;
30    fn find_entity_by_name(&self, name: &str) -> Result<Option<Entity>>;
31
32    fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
33
34    fn store_relation(&self, relation: &Relation) -> Result<()>;
35    fn get_relations(
36        &self,
37        node_id: Uuid,
38        relation_type: Option<RelationType>,
39    ) -> Result<Vec<Relation>>;
40
41    fn vector_search(&self, embedding: &[f32], limit: usize) -> Result<Vec<(Memory, f32)>>;
42
43    /// Cosine search restricted to memories matching `source`, `memory_type`,
44    /// and/or `project_path`, scanned exhaustively over that subset.
45    ///
46    /// A scoped query cannot be served by post-filtering an unfiltered
47    /// shortlist, because the shortlist is chosen before the filter is known:
48    /// a memory that does not place in the global top-k is then unreachable no
49    /// matter how narrow the filter is. Scoping must constrain generation.
50    fn filtered_vector_search(
51        &self,
52        embedding: &[f32],
53        source: Option<&str>,
54        memory_type: Option<MemoryType>,
55        project_path: Option<&str>,
56        limit: usize,
57    ) -> Result<Vec<(Memory, f32)>>;
58
59    fn traverse(&self, start_id: Uuid, depth: u32) -> Result<Vec<(Memory, Vec<Relation>)>>;
60
61    fn memories_by_source(&self, source: &str) -> Result<Vec<Memory>>;
62    fn memories_by_type(&self, memory_type: MemoryType) -> Result<Vec<Memory>>;
63
64    /// Whether `memory` answers a recall scoped to `source`. Consolidation
65    /// leaves the raws archived and the distillate tagged 'consolidation', so
66    /// a store that tracks distillation must let a distillate answer for the
67    /// sources of its raws, or a source filter finds nothing once its raws
68    /// are consolidated. The default is the plain column comparison for
69    /// stores that never consolidate.
70    fn source_admits(&self, memory: &Memory, source: &str) -> Result<bool> {
71        Ok(memory.source == source)
72    }
73
74    /// The text around a memory's chunk `ord`: that chunk joined with one
75    /// neighbour on each side. None for a store that keeps no chunks.
76    fn chunk_window(&self, memory_id: Uuid, ord: usize) -> Result<Option<String>> {
77        let _ = (memory_id, ord);
78        Ok(None)
79    }
80
81    fn update_memory(&self, memory: &Memory) -> Result<()>;
82
83    fn record_access(&self, memory: &Memory) -> Result<()>;
84
85    /// Count one access on each of `ids` and stamp them with the current
86    /// time; confidence is left alone. A recall touches every result it
87    /// returns, so an implementation should batch these into one write
88    /// rather than take the writer lock per row. Ids the store does not
89    /// hold are ignored.
90    fn record_accesses(&self, ids: &[Uuid]) -> Result<()> {
91        let now = Utc::now();
92        for id in ids {
93            if let Some(mut memory) = self.get_memory(*id)? {
94                memory.access_count += 1;
95                memory.last_accessed = now;
96                self.record_access(&memory)?;
97            }
98        }
99        Ok(())
100    }
101
102    fn bm25_search(&self, query: &str, limit: usize) -> Result<Vec<Memory>>;
103
104    fn memory_count(&self) -> Result<usize>;
105
106    fn all_memory_ids(&self) -> Result<Vec<Uuid>>;
107
108    // Every Memory-rooted edge as a Relation, mirroring get_relations: strength
109    // is always 1.0 and context is always None (get_relations discards edge
110    // properties), so a bulk fetch is interchangeable with per-id reads. Used to
111    // prewarm a relation cache in one scan instead of thousands of point reads.
112    fn all_relations(&self) -> Result<Vec<Relation>>;
113}
114
115/// Live consolidation state, read from the memories table rather than the
116/// append-only distillation_log, whose row count keeps counting raws that
117/// were purged long ago.
118#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
119pub struct ConsolidationStats {
120    /// Hot, unsuperseded memories with source = 'consolidation'.
121    pub distilled_live: u64,
122    /// Hot, unsuperseded raws that no distillation pass has seen yet.
123    pub raw_pending: u64,
124    /// Distilled or skipped raws already moved to the archive tier.
125    pub raw_retired: u64,
126    /// Distilled or skipped raws still hot; the next retire pass moves them.
127    pub raw_retire_pending: u64,
128}
129
130/// Raws waiting for a distillation pass, by the same rule as
131/// `ConsolidationStats::raw_pending`.
132#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
133pub struct PendingDistillation {
134    pub count: u64,
135    pub oldest_created_at: Option<DateTime<Utc>>,
136}
137
138/// What `SqliteStore::restore_explicit_memories` changed.
139#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
140pub struct ExplicitRestoreStats {
141    /// Archived explicit raws moved back to the hot tier.
142    pub raws_restored: u64,
143    /// distillation_log rows removed from explicit raws.
144    pub markers_dropped: u64,
145    /// Distillates whose raws were all explicit, now superseded.
146    pub distillates_superseded: u64,
147}
148
149/// What `SqliteStore::refile_stray_project_slugs` changed and what it
150/// declined to decide.
151#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
152pub struct StraySlugReport {
153    pub memories_refiled: u64,
154    pub refiled: Vec<RefiledSlug>,
155    /// Names that end more than one stored project; left where they are.
156    pub ambiguous: Vec<AmbiguousSlug>,
157    /// Names that end no stored project; left where they are.
158    pub unresolved: Vec<String>,
159}
160
161#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
162pub struct RefiledSlug {
163    pub from: String,
164    pub to: String,
165    pub memories: u64,
166}
167
168#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
169pub struct AmbiguousSlug {
170    pub slug: String,
171    pub candidates: Vec<String>,
172}
173
174#[derive(Debug, Clone, Copy, PartialEq, Eq)]
175pub enum ApplyOutcome {
176    Created,
177    Updated,
178    Deleted,
179    Skipped(SkipReason),
180}
181
182/// Why a sync record was not applied. Carried on the outcome because a bare
183/// skip count cannot distinguish routine convergence (the row is already here)
184/// from silent data loss (the edge can never resolve), and that ambiguity hid
185/// tens of thousands of dropped relations for weeks.
186#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
187#[serde(rename_all = "snake_case")]
188pub enum SkipReason {
189    AlreadyPresent,
190    Tombstoned,
191    DanglingEndpoint,
192    StaleWrite,
193    Unsupported,
194}
195
196impl SkipReason {
197    pub fn as_str(self) -> &'static str {
198        match self {
199            SkipReason::AlreadyPresent => "already_present",
200            SkipReason::Tombstoned => "tombstoned",
201            SkipReason::DanglingEndpoint => "dangling_endpoint",
202            SkipReason::StaleWrite => "stale_write",
203            SkipReason::Unsupported => "unsupported",
204        }
205    }
206}
207
208/// The write/admin surface behind the daemon, sync, and maintenance CLI.
209/// Split from Store so the query path stays lean enough for stub-store tests.
210pub trait StoreAdmin: Store {
211    fn register_machine(&self, identity: &MachineIdentity) -> Result<()>;
212    fn get_all_machines(&self) -> Result<HashMap<String, String>>;
213    fn diagnostic(&self) -> Result<String>;
214
215    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
216    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
217    /// Writes `memory` only when no memory holds its id in any state and the
218    /// id is not tombstoned, checked in the insert's own transaction so two
219    /// concurrent writers cannot both pass. True when it wrote. The default
220    /// cannot check and always writes.
221    fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
222        self.store_memory(memory)?;
223        Ok(true)
224    }
225    /// `store_memory_if_absent` for conversations: an existing row keeps its
226    /// fields and nothing is logged for sync.
227    fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
228        self.store_conversation(conversation)?;
229        Ok(true)
230    }
231    /// Whether a memory holds `id` in any state or the id is tombstoned. A
232    /// read outside any write, so it can only spare work ahead of
233    /// `store_memory_if_absent`, never replace it. The default knows nothing.
234    fn memory_id_taken(&self, _id: Uuid) -> Result<bool> {
235        Ok(false)
236    }
237    fn is_file_changed(&self, file_path: &str, file_hash: &str) -> Result<bool>;
238    fn mark_ingested(
239        &self,
240        file_path: &str,
241        file_hash: &str,
242        memory_count: usize,
243        source: &str,
244    ) -> Result<()>;
245    fn ingested_file_count(&self) -> Result<usize>;
246    fn clear_ingest_log(&self) -> Result<usize>;
247    fn ingest_log_entry(&self, _file_path: &str) -> Result<Option<IngestLogEntry>> {
248        Ok(None)
249    }
250    /// Any row, at any path, recording a file with this session key and
251    /// content hash.
252    fn ingested_session(
253        &self,
254        _session_key: &str,
255        _content_hash: &str,
256    ) -> Result<Option<IngestLogEntry>> {
257        Ok(None)
258    }
259    /// Legacy rows still holding an old-format hash whose file stem is
260    /// exactly `stem`.
261    fn legacy_ingest_entries_with_stem(&self, _stem: &str) -> Result<Vec<IngestLogEntry>> {
262        Ok(Vec::new())
263    }
264    /// Replaces a row's hash with its SHA-256 and session key in place,
265    /// keeping its legacy flag.
266    fn upgrade_ingest_hash(
267        &self,
268        _file_path: &str,
269        _content_hash: &str,
270        _session_key: Option<&str>,
271    ) -> Result<()> {
272        Ok(())
273    }
274    fn record_ingested(&self, record: &IngestRecord<'_>) -> Result<()> {
275        self.mark_ingested(
276            record.file_path,
277            record.content_hash,
278            record.memory_count,
279            record.source,
280        )
281    }
282    /// Clears the rows a reingest may re-parse, keeping legacy rows: their
283    /// raws deleted under random ids would come back under deterministic ones.
284    fn clear_reparseable_ingest_log(&self) -> Result<usize> {
285        self.clear_ingest_log()
286    }
287    fn delete_memories_by_source(&self, source: &str) -> Result<usize>;
288    fn delete_memories_by_source_id(&self, source_id: &str) -> Result<usize>;
289    fn rebuild_vector_index(&self) -> Result<()>;
290    fn backfill_project_paths(&self) -> Result<u64>;
291    fn backfill_sync_log(&self) -> Result<u64>;
292    fn backfill_tombstones(&self) -> Result<u64>;
293    fn prune_dangling_relations(&self) -> Result<u64>;
294    /// Collapses re-logged duplicate `sync_log` rows down to one per
295    /// `(node_type, node_id, op)`, keeping the highest `local_seq` in each
296    /// group since the counter is derived as `MAX(local_seq)+1` on every
297    /// write: dropping the row holding the true max would let the next
298    /// write reuse a sequence a peer already considers seen.
299    fn compact_sync_log(&self, ops: &[SyncOp], dry_run: bool) -> Result<CompactStats>;
300    fn memories_by_project_path(&self, project_path: &str) -> Result<Vec<Memory>>;
301    /// Every distinct non-null `project_path` value with its live memory
302    /// count, so a caller can discover the real slug `recall`'s `project`
303    /// scope expects instead of guessing.
304    fn distinct_project_paths(&self) -> Result<Vec<(String, u64)>>;
305    fn all_memories_with_embeddings(&self) -> Result<Vec<Memory>>;
306
307    fn consolidation_stats(&self) -> Result<ConsolidationStats>;
308    /// Hot, unsuperseded raws with no distillation_log row, ordered by
309    /// (source_id, created_at) so a batch reads as conversations.
310    fn unconsolidated_memories(&self, limit: usize) -> Result<Vec<Memory>>;
311    fn mark_consolidated(&self, raw_id: Uuid, distilled_id: Uuid, model: &str) -> Result<()>;
312    /// Moves every distilled or skip-marked raw that is still hot to the
313    /// archive tier. Nothing is deleted: the raw keeps its content and
314    /// embedding and only leaves recall, so a distillate can always be
315    /// traced back to what it was distilled from.
316    fn retire_consolidated_raws(&self) -> Result<usize>;
317    fn memories_created_between(
318        &self,
319        start: &DateTime<Utc>,
320        end: &DateTime<Utc>,
321    ) -> Result<Vec<Memory>>;
322
323    fn sync_log_since(&self, after_seq: u64) -> Result<Vec<SyncEntry>>;
324    fn sync_log_page(&self, after_seq: u64, limit: Option<usize>) -> Result<Vec<SyncEntry>>;
325    fn max_sync_seq(&self) -> Result<u64>;
326    fn get_sync_state(&self, peer_id: &str) -> Result<Option<SyncState>>;
327    fn set_sync_state(&self, state: &SyncState) -> Result<()>;
328    fn get_all_sync_states(&self) -> Result<Vec<SyncState>>;
329    fn tombstone_exists(&self, id: Uuid) -> Result<bool>;
330
331    /// Applies one remote sync record with LWW/tombstone conflict semantics,
332    /// in its own transaction.
333    fn apply_record(&self, record: &SyncRecord) -> Result<ApplyOutcome>;
334
335    /// Applies a batch atomically: either every record commits or none do, so
336    /// a caller can fall back to per-record apply_record on error without
337    /// double-applying a prefix.
338    fn apply_batch(&self, records: &[SyncRecord]) -> Result<Vec<ApplyOutcome>>;
339}