1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
use std::collections::HashMap;
use anyhow::Result;
use chrono::{DateTime, Utc};
use uuid::Uuid;
use crate::machine::MachineIdentity;
use crate::schema::{
CompactStats, Conversation, Entity, IngestLogEntry, IngestRecord, Memory, MemoryType, Relation,
RelationType, SyncEntry, SyncOp, SyncRecord, SyncState,
};
pub trait Store: Send + Sync {
fn store_memory(&self, memory: &Memory) -> Result<()>;
fn get_memory(&self, id: Uuid) -> Result<Option<Memory>>;
fn get_memory_with_embedding(&self, id: Uuid) -> Result<Option<Memory>>;
fn delete_memory(&self, id: Uuid) -> Result<()>;
/// `delete_memory` that also reports whether a row was there to remove,
/// so a forget of an unknown id can be answered as such instead of
/// silently succeeding.
fn remove_memory(&self, id: Uuid) -> Result<bool> {
let existed = self.get_memory(id)?.is_some();
self.delete_memory(id)?;
Ok(existed)
}
fn store_entity(&self, entity: &Entity) -> Result<()>;
fn get_entity(&self, id: Uuid) -> Result<Option<Entity>>;
fn find_entity_by_name(&self, name: &str) -> Result<Option<Entity>>;
fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
fn store_relation(&self, relation: &Relation) -> Result<()>;
fn get_relations(
&self,
node_id: Uuid,
relation_type: Option<RelationType>,
) -> Result<Vec<Relation>>;
fn vector_search(&self, embedding: &[f32], limit: usize) -> Result<Vec<(Memory, f32)>>;
/// Cosine search restricted to memories matching `source`, `memory_type`,
/// and/or `project_path`, scanned exhaustively over that subset.
///
/// A scoped query cannot be served by post-filtering an unfiltered
/// shortlist, because the shortlist is chosen before the filter is known:
/// a memory that does not place in the global top-k is then unreachable no
/// matter how narrow the filter is. Scoping must constrain generation.
fn filtered_vector_search(
&self,
embedding: &[f32],
source: Option<&str>,
memory_type: Option<MemoryType>,
project_path: Option<&str>,
limit: usize,
) -> Result<Vec<(Memory, f32)>>;
fn traverse(&self, start_id: Uuid, depth: u32) -> Result<Vec<(Memory, Vec<Relation>)>>;
fn memories_by_source(&self, source: &str) -> Result<Vec<Memory>>;
fn memories_by_type(&self, memory_type: MemoryType) -> Result<Vec<Memory>>;
/// Whether `memory` answers a recall scoped to `source`. Consolidation
/// leaves the raws archived and the distillate tagged 'consolidation', so
/// a store that tracks distillation must let a distillate answer for the
/// sources of its raws, or a source filter finds nothing once its raws
/// are consolidated. The default is the plain column comparison for
/// stores that never consolidate.
fn source_admits(&self, memory: &Memory, source: &str) -> Result<bool> {
Ok(memory.source == source)
}
/// The text around a memory's chunk `ord`: that chunk joined with one
/// neighbour on each side. None for a store that keeps no chunks.
fn chunk_window(&self, memory_id: Uuid, ord: usize) -> Result<Option<String>> {
let _ = (memory_id, ord);
Ok(None)
}
fn update_memory(&self, memory: &Memory) -> Result<()>;
fn record_access(&self, memory: &Memory) -> Result<()>;
/// Count one access on each of `ids` and stamp them with the current
/// time; confidence is left alone. A recall touches every result it
/// returns, so an implementation should batch these into one write
/// rather than take the writer lock per row. Ids the store does not
/// hold are ignored.
fn record_accesses(&self, ids: &[Uuid]) -> Result<()> {
let now = Utc::now();
for id in ids {
if let Some(mut memory) = self.get_memory(*id)? {
memory.access_count += 1;
memory.last_accessed = now;
self.record_access(&memory)?;
}
}
Ok(())
}
fn bm25_search(&self, query: &str, limit: usize) -> Result<Vec<Memory>>;
fn memory_count(&self) -> Result<usize>;
fn all_memory_ids(&self) -> Result<Vec<Uuid>>;
// Every Memory-rooted edge as a Relation, mirroring get_relations: strength
// is always 1.0 and context is always None (get_relations discards edge
// properties), so a bulk fetch is interchangeable with per-id reads. Used to
// prewarm a relation cache in one scan instead of thousands of point reads.
fn all_relations(&self) -> Result<Vec<Relation>>;
}
/// Live consolidation state, read from the memories table rather than the
/// append-only distillation_log, whose row count keeps counting raws that
/// were purged long ago.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ConsolidationStats {
/// Hot, unsuperseded memories with source = 'consolidation'.
pub distilled_live: u64,
/// Hot, unsuperseded raws that no distillation pass has seen yet.
pub raw_pending: u64,
/// Distilled or skipped raws already moved to the archive tier.
pub raw_retired: u64,
/// Distilled or skipped raws still hot; the next retire pass moves them.
pub raw_retire_pending: u64,
}
/// Raws waiting for a distillation pass, by the same rule as
/// `ConsolidationStats::raw_pending`.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct PendingDistillation {
pub count: u64,
pub oldest_created_at: Option<DateTime<Utc>>,
}
/// What `SqliteStore::restore_explicit_memories` changed.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ExplicitRestoreStats {
/// Archived explicit raws moved back to the hot tier.
pub raws_restored: u64,
/// distillation_log rows removed from explicit raws.
pub markers_dropped: u64,
/// Distillates whose raws were all explicit, now superseded.
pub distillates_superseded: u64,
}
/// What `SqliteStore::refile_stray_project_slugs` changed and what it
/// declined to decide.
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct StraySlugReport {
pub memories_refiled: u64,
pub refiled: Vec<RefiledSlug>,
/// Names that end more than one stored project; left where they are.
pub ambiguous: Vec<AmbiguousSlug>,
/// Names that end no stored project; left where they are.
pub unresolved: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct RefiledSlug {
pub from: String,
pub to: String,
pub memories: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct AmbiguousSlug {
pub slug: String,
pub candidates: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ApplyOutcome {
Created,
Updated,
Deleted,
Skipped(SkipReason),
}
/// Why a sync record was not applied. Carried on the outcome because a bare
/// skip count cannot distinguish routine convergence (the row is already here)
/// from silent data loss (the edge can never resolve), and that ambiguity hid
/// tens of thousands of dropped relations for weeks.
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SkipReason {
AlreadyPresent,
Tombstoned,
DanglingEndpoint,
StaleWrite,
Unsupported,
}
impl SkipReason {
pub fn as_str(self) -> &'static str {
match self {
SkipReason::AlreadyPresent => "already_present",
SkipReason::Tombstoned => "tombstoned",
SkipReason::DanglingEndpoint => "dangling_endpoint",
SkipReason::StaleWrite => "stale_write",
SkipReason::Unsupported => "unsupported",
}
}
}
/// How many leading characters ingest compares to find a duplicate.
pub const CONTENT_PREFIX_CHARS: usize = 128;
/// The write/admin surface behind the daemon, sync, and maintenance CLI.
/// Split from Store so the query path stays lean enough for stub-store tests.
pub trait StoreAdmin: Store {
fn register_machine(&self, identity: &MachineIdentity) -> Result<()>;
fn get_all_machines(&self) -> Result<HashMap<String, String>>;
fn diagnostic(&self) -> Result<String>;
fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
/// Writes `memory` only when no memory holds its id in any state and the
/// id is not tombstoned, checked in the insert's own transaction so two
/// concurrent writers cannot both pass. True when it wrote. The default
/// cannot check and always writes.
fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
self.store_memory(memory)?;
Ok(true)
}
/// `store_memory_if_absent` for ingest, which may also refuse text a
/// delete removed from the same source. The default refuses nothing more.
fn ingest_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
self.store_memory_if_absent(memory)
}
/// Whether ingest must skip `memory` because a delete removed its text
/// from its source. A read outside any write, so it only spares work
/// ahead of `ingest_memory_if_absent`. The default knows of no deletes.
fn ingest_blocked(&self, _memory: &Memory) -> Result<bool> {
Ok(false)
}
/// `store_memory_if_absent` for conversations: an existing row keeps its
/// fields and nothing is logged for sync.
fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
self.store_conversation(conversation)?;
Ok(true)
}
/// Whether a memory holds `id` in any state or the id is tombstoned. A
/// read outside any write, so it can only spare work ahead of
/// `store_memory_if_absent`, never replace it. The default knows nothing.
fn memory_id_taken(&self, _id: Uuid) -> Result<bool> {
Ok(false)
}
/// Whether the Claude Code session the transcript at `path` belongs to
/// was deleted, so nothing of it may be captured again. The default
/// knows of no deletes.
fn session_deleted(&self, _path: &str) -> Result<bool> {
Ok(false)
}
fn is_file_changed(&self, file_path: &str, file_hash: &str) -> Result<bool>;
fn mark_ingested(
&self,
file_path: &str,
file_hash: &str,
memory_count: usize,
source: &str,
) -> Result<()>;
fn ingested_file_count(&self) -> Result<usize>;
fn clear_ingest_log(&self) -> Result<usize>;
fn ingest_log_entry(&self, _file_path: &str) -> Result<Option<IngestLogEntry>> {
Ok(None)
}
/// Any row, at any path, recording a file with this session key and
/// content hash.
fn ingested_session(
&self,
_session_key: &str,
_content_hash: &str,
) -> Result<Option<IngestLogEntry>> {
Ok(None)
}
/// Legacy rows still holding an old-format hash whose file stem is
/// exactly `stem`.
fn legacy_ingest_entries_with_stem(&self, _stem: &str) -> Result<Vec<IngestLogEntry>> {
Ok(Vec::new())
}
/// Replaces a row's hash with its SHA-256 and session key in place,
/// keeping its legacy flag.
fn upgrade_ingest_hash(
&self,
_file_path: &str,
_content_hash: &str,
_session_key: Option<&str>,
) -> Result<()> {
Ok(())
}
fn record_ingested(&self, record: &IngestRecord<'_>) -> Result<()> {
self.mark_ingested(
record.file_path,
record.content_hash,
record.memory_count,
record.source,
)
}
/// Clears the rows a reingest may re-parse, keeping legacy rows: their
/// raws deleted under random ids would come back under deterministic ones.
fn clear_reparseable_ingest_log(&self) -> Result<usize> {
self.clear_ingest_log()
}
fn delete_memories_by_source(&self, source: &str) -> Result<usize>;
fn delete_memories_by_source_id(&self, source_id: &str) -> Result<usize>;
fn rebuild_vector_index(&self) -> Result<()>;
fn backfill_project_paths(&self) -> Result<u64>;
fn backfill_sync_log(&self) -> Result<u64>;
fn backfill_tombstones(&self) -> Result<u64>;
fn prune_dangling_relations(&self) -> Result<u64>;
/// Collapses re-logged duplicate `sync_log` rows down to one per
/// `(node_type, node_id, op)`, keeping the highest `local_seq` in each
/// group.
fn compact_sync_log(&self, ops: &[SyncOp], dry_run: bool) -> Result<CompactStats>;
fn memories_by_project_path(&self, project_path: &str) -> Result<Vec<Memory>>;
/// Every distinct non-null `project_path` value with its live memory
/// count, so a caller can discover the real slug `recall`'s `project`
/// scope expects instead of guessing.
fn distinct_project_paths(&self) -> Result<Vec<(String, u64)>>;
fn all_memories_with_embeddings(&self) -> Result<Vec<Memory>>;
fn consolidation_stats(&self) -> Result<ConsolidationStats>;
/// Hot, unsuperseded raws with no distillation_log row, ordered by
/// (source_id, created_at) so a batch reads as conversations.
fn unconsolidated_memories(&self, limit: usize) -> Result<Vec<Memory>>;
fn mark_consolidated(&self, raw_id: Uuid, distilled_id: Uuid, model: &str) -> Result<()>;
/// Moves every distilled or skip-marked raw that is still hot to the
/// archive tier. Nothing is deleted: the raw keeps its content and
/// embedding and only leaves recall, so a distillate can always be
/// traced back to what it was distilled from.
fn retire_consolidated_raws(&self) -> Result<usize>;
fn memories_created_between(
&self,
start: &DateTime<Utc>,
end: &DateTime<Utc>,
) -> Result<Vec<Memory>>;
fn sync_log_since(&self, after_seq: u64) -> Result<Vec<SyncEntry>>;
fn sync_log_page(&self, after_seq: u64, limit: Option<usize>) -> Result<Vec<SyncEntry>>;
/// A page to send a peer. A store that truncates its log refuses a page
/// that starts below its floor.
fn sync_log_export_page(&self, after_seq: u64, limit: Option<usize>) -> Result<Vec<SyncEntry>> {
self.sync_log_page(after_seq, limit)
}
fn max_sync_seq(&self) -> Result<u64>;
fn get_sync_state(&self, peer_id: &str) -> Result<Option<SyncState>>;
fn set_sync_state(&self, state: &SyncState) -> Result<()>;
fn get_all_sync_states(&self) -> Result<Vec<SyncState>>;
fn tombstone_exists(&self, id: Uuid) -> Result<bool>;
/// Applies one remote sync record with LWW/tombstone conflict semantics,
/// in its own transaction.
fn apply_record(&self, record: &SyncRecord) -> Result<ApplyOutcome>;
/// Applies a batch atomically: either every record commits or none do, so
/// a caller can fall back to per-record apply_record on error without
/// double-applying a prefix.
fn apply_batch(&self, records: &[SyncRecord]) -> Result<Vec<ApplyOutcome>>;
}