Skip to main content

second_brain_core/
store.rs

1use std::collections::HashMap;
2
3use anyhow::Result;
4use chrono::{DateTime, Utc};
5use uuid::Uuid;
6
7use crate::machine::MachineIdentity;
8use crate::schema::{
9    CompactStats, Conversation, Entity, IngestLogEntry, IngestRecord, Memory, MemoryType, Relation,
10    RelationType, SyncEntry, SyncOp, SyncRecord, SyncState,
11};
12
13pub trait Store: Send + Sync {
14    fn store_memory(&self, memory: &Memory) -> Result<()>;
15    fn get_memory(&self, id: Uuid) -> Result<Option<Memory>>;
16    fn get_memory_with_embedding(&self, id: Uuid) -> Result<Option<Memory>>;
17    fn delete_memory(&self, id: Uuid) -> Result<()>;
18
19    /// `delete_memory` that also reports whether a row was there to remove,
20    /// so a forget of an unknown id can be answered as such instead of
21    /// silently succeeding.
22    fn remove_memory(&self, id: Uuid) -> Result<bool> {
23        let existed = self.get_memory(id)?.is_some();
24        self.delete_memory(id)?;
25        Ok(existed)
26    }
27
28    fn store_entity(&self, entity: &Entity) -> Result<()>;
29    fn get_entity(&self, id: Uuid) -> Result<Option<Entity>>;
30    fn find_entity_by_name(&self, name: &str) -> Result<Option<Entity>>;
31
32    fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
33
34    fn store_relation(&self, relation: &Relation) -> Result<()>;
35    fn get_relations(
36        &self,
37        node_id: Uuid,
38        relation_type: Option<RelationType>,
39    ) -> Result<Vec<Relation>>;
40
41    fn vector_search(&self, embedding: &[f32], limit: usize) -> Result<Vec<(Memory, f32)>>;
42
43    /// Cosine search restricted to memories matching `source`, `memory_type`,
44    /// and/or `project_path`, scanned exhaustively over that subset.
45    ///
46    /// A scoped query cannot be served by post-filtering an unfiltered
47    /// shortlist, because the shortlist is chosen before the filter is known:
48    /// a memory that does not place in the global top-k is then unreachable no
49    /// matter how narrow the filter is. Scoping must constrain generation.
50    fn filtered_vector_search(
51        &self,
52        embedding: &[f32],
53        source: Option<&str>,
54        memory_type: Option<MemoryType>,
55        project_path: Option<&str>,
56        limit: usize,
57    ) -> Result<Vec<(Memory, f32)>>;
58
59    fn traverse(&self, start_id: Uuid, depth: u32) -> Result<Vec<(Memory, Vec<Relation>)>>;
60
61    fn memories_by_source(&self, source: &str) -> Result<Vec<Memory>>;
62    fn memories_by_type(&self, memory_type: MemoryType) -> Result<Vec<Memory>>;
63
64    /// Whether `memory` answers a recall scoped to `source`. Consolidation
65    /// leaves the raws archived and the distillate tagged 'consolidation', so
66    /// a store that tracks distillation must let a distillate answer for the
67    /// sources of its raws, or a source filter finds nothing once its raws
68    /// are consolidated. The default is the plain column comparison for
69    /// stores that never consolidate.
70    fn source_admits(&self, memory: &Memory, source: &str) -> Result<bool> {
71        Ok(memory.source == source)
72    }
73
74    /// The text around a memory's chunk `ord`: that chunk joined with one
75    /// neighbour on each side. None for a store that keeps no chunks.
76    fn chunk_window(&self, memory_id: Uuid, ord: usize) -> Result<Option<String>> {
77        let _ = (memory_id, ord);
78        Ok(None)
79    }
80
81    fn update_memory(&self, memory: &Memory) -> Result<()>;
82
83    fn record_access(&self, memory: &Memory) -> Result<()>;
84
85    /// Count one access on each of `ids` and stamp them with the current
86    /// time; confidence is left alone. A recall touches every result it
87    /// returns, so an implementation should batch these into one write
88    /// rather than take the writer lock per row. Ids the store does not
89    /// hold are ignored.
90    fn record_accesses(&self, ids: &[Uuid]) -> Result<()> {
91        let now = Utc::now();
92        for id in ids {
93            if let Some(mut memory) = self.get_memory(*id)? {
94                memory.access_count += 1;
95                memory.last_accessed = now;
96                self.record_access(&memory)?;
97            }
98        }
99        Ok(())
100    }
101
102    fn bm25_search(&self, query: &str, limit: usize) -> Result<Vec<Memory>>;
103
104    fn memory_count(&self) -> Result<usize>;
105
106    fn all_memory_ids(&self) -> Result<Vec<Uuid>>;
107
108    // Every Memory-rooted edge as a Relation, mirroring get_relations: strength
109    // is always 1.0 and context is always None (get_relations discards edge
110    // properties), so a bulk fetch is interchangeable with per-id reads. Used to
111    // prewarm a relation cache in one scan instead of thousands of point reads.
112    fn all_relations(&self) -> Result<Vec<Relation>>;
113}
114
115/// Live consolidation state, read from the memories table rather than the
116/// append-only distillation_log, whose row count keeps counting raws that
117/// were purged long ago.
118#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
119pub struct ConsolidationStats {
120    /// Hot, unsuperseded memories with source = 'consolidation'.
121    pub distilled_live: u64,
122    /// Hot, unsuperseded raws that no distillation pass has seen yet.
123    pub raw_pending: u64,
124    /// Distilled or skipped raws already moved to the archive tier.
125    pub raw_retired: u64,
126    /// Distilled or skipped raws still hot; the next retire pass moves them.
127    pub raw_retire_pending: u64,
128}
129
130/// Raws waiting for a distillation pass, by the same rule as
131/// `ConsolidationStats::raw_pending`.
132#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
133pub struct PendingDistillation {
134    pub count: u64,
135    pub oldest_created_at: Option<DateTime<Utc>>,
136}
137
138/// What `SqliteStore::restore_explicit_memories` changed.
139#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
140pub struct ExplicitRestoreStats {
141    /// Archived explicit raws moved back to the hot tier.
142    pub raws_restored: u64,
143    /// distillation_log rows removed from explicit raws.
144    pub markers_dropped: u64,
145    /// Distillates whose raws were all explicit, now superseded.
146    pub distillates_superseded: u64,
147}
148
149/// What `SqliteStore::refile_stray_project_slugs` changed and what it
150/// declined to decide.
151#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
152pub struct StraySlugReport {
153    pub memories_refiled: u64,
154    pub refiled: Vec<RefiledSlug>,
155    /// Names that end more than one stored project; left where they are.
156    pub ambiguous: Vec<AmbiguousSlug>,
157    /// Names that end no stored project; left where they are.
158    pub unresolved: Vec<String>,
159}
160
161#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
162pub struct RefiledSlug {
163    pub from: String,
164    pub to: String,
165    pub memories: u64,
166}
167
168#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
169pub struct AmbiguousSlug {
170    pub slug: String,
171    pub candidates: Vec<String>,
172}
173
174#[derive(Debug, Clone, Copy, PartialEq, Eq)]
175pub enum ApplyOutcome {
176    Created,
177    Updated,
178    Deleted,
179    Skipped(SkipReason),
180}
181
182/// Why a sync record was not applied. Carried on the outcome because a bare
183/// skip count cannot distinguish routine convergence (the row is already here)
184/// from silent data loss (the edge can never resolve), and that ambiguity hid
185/// tens of thousands of dropped relations for weeks.
186#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
187#[serde(rename_all = "snake_case")]
188pub enum SkipReason {
189    AlreadyPresent,
190    Tombstoned,
191    DanglingEndpoint,
192    StaleWrite,
193    Unsupported,
194}
195
196impl SkipReason {
197    pub fn as_str(self) -> &'static str {
198        match self {
199            SkipReason::AlreadyPresent => "already_present",
200            SkipReason::Tombstoned => "tombstoned",
201            SkipReason::DanglingEndpoint => "dangling_endpoint",
202            SkipReason::StaleWrite => "stale_write",
203            SkipReason::Unsupported => "unsupported",
204        }
205    }
206}
207
208/// How many leading characters ingest compares to find a duplicate.
209pub const CONTENT_PREFIX_CHARS: usize = 128;
210
211/// The write/admin surface behind the daemon, sync, and maintenance CLI.
212/// Split from Store so the query path stays lean enough for stub-store tests.
213pub trait StoreAdmin: Store {
214    fn register_machine(&self, identity: &MachineIdentity) -> Result<()>;
215    fn get_all_machines(&self) -> Result<HashMap<String, String>>;
216    fn diagnostic(&self) -> Result<String>;
217
218    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
219    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
220    /// Writes `memory` only when no memory holds its id in any state and the
221    /// id is not tombstoned, checked in the insert's own transaction so two
222    /// concurrent writers cannot both pass. True when it wrote. The default
223    /// cannot check and always writes.
224    fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
225        self.store_memory(memory)?;
226        Ok(true)
227    }
228    /// `store_memory_if_absent` for ingest, which may also refuse text a
229    /// delete removed from the same source. The default refuses nothing more.
230    fn ingest_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
231        self.store_memory_if_absent(memory)
232    }
233    /// Whether ingest must skip `memory` because a delete removed its text
234    /// from its source. A read outside any write, so it only spares work
235    /// ahead of `ingest_memory_if_absent`. The default knows of no deletes.
236    fn ingest_blocked(&self, _memory: &Memory) -> Result<bool> {
237        Ok(false)
238    }
239    /// `store_memory_if_absent` for conversations: an existing row keeps its
240    /// fields and nothing is logged for sync.
241    fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
242        self.store_conversation(conversation)?;
243        Ok(true)
244    }
245    /// Whether a memory holds `id` in any state or the id is tombstoned. A
246    /// read outside any write, so it can only spare work ahead of
247    /// `store_memory_if_absent`, never replace it. The default knows nothing.
248    fn memory_id_taken(&self, _id: Uuid) -> Result<bool> {
249        Ok(false)
250    }
251    /// Whether the Claude Code session the transcript at `path` belongs to
252    /// was deleted, so nothing of it may be captured again. The default
253    /// knows of no deletes.
254    fn session_deleted(&self, _path: &str) -> Result<bool> {
255        Ok(false)
256    }
257    fn is_file_changed(&self, file_path: &str, file_hash: &str) -> Result<bool>;
258    fn mark_ingested(
259        &self,
260        file_path: &str,
261        file_hash: &str,
262        memory_count: usize,
263        source: &str,
264    ) -> Result<()>;
265    fn ingested_file_count(&self) -> Result<usize>;
266    fn clear_ingest_log(&self) -> Result<usize>;
267    fn ingest_log_entry(&self, _file_path: &str) -> Result<Option<IngestLogEntry>> {
268        Ok(None)
269    }
270    /// Any row, at any path, recording a file with this session key and
271    /// content hash.
272    fn ingested_session(
273        &self,
274        _session_key: &str,
275        _content_hash: &str,
276    ) -> Result<Option<IngestLogEntry>> {
277        Ok(None)
278    }
279    /// Legacy rows still holding an old-format hash whose file stem is
280    /// exactly `stem`.
281    fn legacy_ingest_entries_with_stem(&self, _stem: &str) -> Result<Vec<IngestLogEntry>> {
282        Ok(Vec::new())
283    }
284    /// Replaces a row's hash with its SHA-256 and session key in place,
285    /// keeping its legacy flag.
286    fn upgrade_ingest_hash(
287        &self,
288        _file_path: &str,
289        _content_hash: &str,
290        _session_key: Option<&str>,
291    ) -> Result<()> {
292        Ok(())
293    }
294    fn record_ingested(&self, record: &IngestRecord<'_>) -> Result<()> {
295        self.mark_ingested(
296            record.file_path,
297            record.content_hash,
298            record.memory_count,
299            record.source,
300        )
301    }
302    /// Clears the rows a reingest may re-parse, keeping legacy rows: their
303    /// raws deleted under random ids would come back under deterministic ones.
304    fn clear_reparseable_ingest_log(&self) -> Result<usize> {
305        self.clear_ingest_log()
306    }
307    fn delete_memories_by_source(&self, source: &str) -> Result<usize>;
308    fn delete_memories_by_source_id(&self, source_id: &str) -> Result<usize>;
309    fn rebuild_vector_index(&self) -> Result<()>;
310    fn backfill_project_paths(&self) -> Result<u64>;
311    fn backfill_sync_log(&self) -> Result<u64>;
312    fn backfill_tombstones(&self) -> Result<u64>;
313    fn prune_dangling_relations(&self) -> Result<u64>;
314    /// Collapses re-logged duplicate `sync_log` rows down to one per
315    /// `(node_type, node_id, op)`, keeping the highest `local_seq` in each
316    /// group.
317    fn compact_sync_log(&self, ops: &[SyncOp], dry_run: bool) -> Result<CompactStats>;
318    fn memories_by_project_path(&self, project_path: &str) -> Result<Vec<Memory>>;
319    /// Every distinct non-null `project_path` value with its live memory
320    /// count, so a caller can discover the real slug `recall`'s `project`
321    /// scope expects instead of guessing.
322    fn distinct_project_paths(&self) -> Result<Vec<(String, u64)>>;
323    fn all_memories_with_embeddings(&self) -> Result<Vec<Memory>>;
324
325    fn consolidation_stats(&self) -> Result<ConsolidationStats>;
326    /// Hot, unsuperseded raws with no distillation_log row, ordered by
327    /// (source_id, created_at) so a batch reads as conversations.
328    fn unconsolidated_memories(&self, limit: usize) -> Result<Vec<Memory>>;
329    fn mark_consolidated(&self, raw_id: Uuid, distilled_id: Uuid, model: &str) -> Result<()>;
330    /// Moves every distilled or skip-marked raw that is still hot to the
331    /// archive tier. Nothing is deleted: the raw keeps its content and
332    /// embedding and only leaves recall, so a distillate can always be
333    /// traced back to what it was distilled from.
334    fn retire_consolidated_raws(&self) -> Result<usize>;
335    fn memories_created_between(
336        &self,
337        start: &DateTime<Utc>,
338        end: &DateTime<Utc>,
339    ) -> Result<Vec<Memory>>;
340
341    fn sync_log_since(&self, after_seq: u64) -> Result<Vec<SyncEntry>>;
342    fn sync_log_page(&self, after_seq: u64, limit: Option<usize>) -> Result<Vec<SyncEntry>>;
343    /// A page to send a peer. A store that truncates its log refuses a page
344    /// that starts below its floor.
345    fn sync_log_export_page(&self, after_seq: u64, limit: Option<usize>) -> Result<Vec<SyncEntry>> {
346        self.sync_log_page(after_seq, limit)
347    }
348    fn max_sync_seq(&self) -> Result<u64>;
349    fn get_sync_state(&self, peer_id: &str) -> Result<Option<SyncState>>;
350    fn set_sync_state(&self, state: &SyncState) -> Result<()>;
351    fn get_all_sync_states(&self) -> Result<Vec<SyncState>>;
352    fn tombstone_exists(&self, id: Uuid) -> Result<bool>;
353
354    /// Applies one remote sync record with LWW/tombstone conflict semantics,
355    /// in its own transaction.
356    fn apply_record(&self, record: &SyncRecord) -> Result<ApplyOutcome>;
357
358    /// Applies a batch atomically: either every record commits or none do, so
359    /// a caller can fall back to per-record apply_record on error without
360    /// double-applying a prefix.
361    fn apply_batch(&self, records: &[SyncRecord]) -> Result<Vec<ApplyOutcome>>;
362}