mnemo_engine/store/threads.rs
1//! Phase 12.7a: the thread archive (ingest, export, forget).
2
3use super::*;
4
5/// One incoming turn after chunking or truncation, ready to embed and
6/// write (Phase 12.7a, split out for 12.7d's batch embed).
7pub(super) struct PendingTurn {
8 pub(super) turn_id: String,
9 pub(super) role: String,
10 pub(super) created_at: i64,
11 pub(super) parts: Vec<String>,
12 /// Hex sha256 + original length, set only for a truncated
13 /// tool-result turn. Applied to the single chunk the truncation
14 /// yielded.
15 pub(super) truncation: Option<(String, u64)>,
16}
17
18/// Output of `Mnemo::plan_ingest`: the turns to write, the first free
19/// `_mnemo.seq`, and the counts gathered so far (skips, chunking,
20/// truncation).
21pub(super) struct IngestPlan {
22 pub(super) pending: Vec<PendingTurn>,
23 pub(super) next_seq: u64,
24 pub(super) report: IngestReport,
25}
26
27impl IngestPlan {
28 /// Every text to embed, in write order (turns, then parts).
29 pub(super) fn texts(&self) -> Vec<&str> {
30 self.pending
31 .iter()
32 .flat_map(|p| p.parts.iter().map(String::as_str))
33 .collect()
34 }
35
36 pub(super) fn text_count(&self) -> usize {
37 self.pending.iter().map(|p| p.parts.len()).sum()
38 }
39}
40
41/// Hex-encode a byte slice. Low-dependency alternative to pulling in a
42/// hex crate for the lone call site in `ingest_thread`'s SHA-256
43/// recording.
44fn hex_encode(bytes: &[u8]) -> String {
45 const HEX: &[u8; 16] = b"0123456789abcdef";
46 let mut out = String::with_capacity(bytes.len() * 2);
47 for b in bytes {
48 out.push(HEX[(b >> 4) as usize] as char);
49 out.push(HEX[(b & 0x0f) as usize] as char);
50 }
51 out
52}
53
54/// Return the longest UTF-8 prefix of `s` whose byte length is `<=
55/// max_bytes`. Always lands on a `char` boundary.
56fn take_utf8_prefix(s: &str, max_bytes: usize) -> String {
57 if s.len() <= max_bytes {
58 return s.to_string();
59 }
60 // Walk forward through char indices; the last index whose byte
61 // offset fits inside `max_bytes` tells us where to cut.
62 let mut cut = 0usize;
63 for (idx, _) in s.char_indices() {
64 if idx > max_bytes {
65 break;
66 }
67 cut = idx;
68 }
69 s[..cut].to_string()
70}
71
72fn chunk_utf8(s: &str, max_bytes: usize) -> Vec<String> {
73 if s.len() <= max_bytes {
74 return vec![s.to_string()];
75 }
76 let mut out = Vec::new();
77 let mut rest = s;
78 while rest.len() > max_bytes {
79 let head = take_utf8_prefix(rest, max_bytes);
80 let head_bytes = head.len();
81 out.push(head);
82 rest = &rest[head_bytes..];
83 }
84 if !rest.is_empty() {
85 out.push(rest.to_string());
86 }
87 out
88}
89
90impl Mnemo {
91 /// Ingest a batch of turns into the archive layer of `thread_id`.
92 ///
93 /// Each [`ThreadTurn`] becomes one or more `Episodic` records
94 /// tagged with the usual thread-archive reserved-metadata set
95 /// (`_mnemo.kind = "turn"`, `_mnemo.thread`, `_mnemo.turn_id`,
96 /// `_mnemo.seq`, `_mnemo.role`). The ingest path is
97 /// **idempotent** per `turn_id`: a turn whose `turn_id` already
98 /// appears on a record in this thread is skipped, so re-running
99 /// `ingest_thread` with an extended transcript appends only
100 /// what's new.
101 ///
102 /// **Chunking.** A non-tool turn whose UTF-8 content exceeds
103 /// [`THREAD_TURN_CHUNK_BYTES`] (8 KB) is split into chunks at
104 /// char boundaries, each stored as its own record with
105 /// `_mnemo.part = 0, 1, …` under the same `turn_id`.
106 ///
107 /// **Truncation.** A `role = "tool"` turn whose content exceeds
108 /// [`THREAD_TOOL_RESULT_TRUNCATE_BYTES`] (16 KB) is stored
109 /// truncated — the first 16 KB of UTF-8 goes into the record,
110 /// and `_mnemo.sha256` + `_mnemo.length` capture the original
111 /// hash and byte length so the full content can be recovered
112 /// (or verified) out-of-band.
113 ///
114 /// **Embedding.** An embedder must be registered
115 /// ([`Mnemo::set_embedder`]); the content strings (one per
116 /// resulting chunk / truncated record) are batched into a single
117 /// `embed()` call.
118 ///
119 /// Call `self.flush()` after a successful ingest to persist.
120 pub fn ingest_thread(&mut self, thread_id: &str, turns: &[ThreadTurn]) -> Result<IngestReport> {
121 if self.read_only {
122 return Err(MnemoError::ReadOnly);
123 }
124 let plan = self.plan_ingest(thread_id, turns)?;
125 if plan.pending.is_empty() {
126 return Ok(plan.report);
127 }
128 // Batched embed: one round trip to the embedder for the whole
129 // ingest, whatever the turn count.
130 let vectors = {
131 let embedder = self.embedder.as_ref().ok_or_else(|| {
132 MnemoError::Invalid(
133 "ingest_thread: no embedder registered on this handle; call set_embedder first"
134 .into(),
135 )
136 })?;
137 embedder.embed(&plan.texts())?
138 };
139 self.check_vectors("ingest_thread", &vectors, plan.text_count())?;
140 self.apply_ingest(thread_id, plan, vectors, None)
141 }
142
143 /// MOBILE-1a: the texts [`Mnemo::ingest_thread`] would embed for
144 /// `turns`, in order (one per new chunk or truncated record; turns
145 /// already in the thread are skipped). With
146 /// [`Mnemo::ingest_thread_with_vectors`] it splits an ingest in two, so
147 /// a caller can run its embedder between the calls without holding
148 /// anything the handle needs, as the UniFFI binding does. Empty when
149 /// nothing is new. Writes nothing.
150 pub fn ingest_thread_texts(
151 &mut self,
152 thread_id: &str,
153 turns: &[ThreadTurn],
154 ) -> Result<Vec<String>> {
155 let plan = self.plan_ingest(thread_id, turns)?;
156 Ok(plan.texts().into_iter().map(str::to_string).collect())
157 }
158
159 /// MOBILE-1a: [`Mnemo::ingest_thread`] with caller-supplied vectors,
160 /// one per text [`Mnemo::ingest_thread_texts`] returned for the same
161 /// `thread_id` and `turns`, in the same order. No embedder is needed.
162 /// The ingest is planned again here; if the thread changed in between
163 /// (so the count no longer matches) it returns
164 /// [`MnemoError::Invalid`] and writes nothing, and the caller can ask
165 /// for the texts again.
166 pub fn ingest_thread_with_vectors(
167 &mut self,
168 thread_id: &str,
169 turns: &[ThreadTurn],
170 vectors: &[Vec<f32>],
171 ) -> Result<IngestReport> {
172 if self.read_only {
173 return Err(MnemoError::ReadOnly);
174 }
175 let plan = self.plan_ingest(thread_id, turns)?;
176 if plan.pending.is_empty() && vectors.is_empty() {
177 return Ok(plan.report);
178 }
179 self.check_vectors("ingest_thread_with_vectors", vectors, plan.text_count())?;
180 self.apply_ingest(thread_id, plan, vectors.to_vec(), None)
181 }
182
183 /// Check a batch of embedder output before anything is written:
184 /// one vector per input, the file's width, finite and non-zero.
185 pub(super) fn check_vectors(
186 &self,
187 ctx: &str,
188 vectors: &[Vec<f32>],
189 expected: usize,
190 ) -> Result<()> {
191 if vectors.len() != expected {
192 return Err(MnemoError::Invalid(format!(
193 "{ctx}: embedder returned {} vectors for {expected} texts",
194 vectors.len()
195 )));
196 }
197 for v in vectors {
198 self.check_vector(ctx, v)?;
199 }
200 Ok(())
201 }
202
203 /// The checks `put` applies to one vector, run up front so a batch
204 /// can't fail half-way through its writes.
205 pub(super) fn check_vector(&self, ctx: &str, v: &[f32]) -> Result<()> {
206 if v.len() != self.dimensions {
207 return Err(MnemoError::DimensionMismatch {
208 expected: self.dimensions,
209 got: v.len(),
210 });
211 }
212 if embed::is_zero_vector(v) {
213 return Err(MnemoError::Invalid(format!(
214 "{ctx}: all-zero vector refused"
215 )));
216 }
217 if v.iter().any(|x| !x.is_finite()) {
218 return Err(MnemoError::Invalid(format!(
219 "{ctx}: vector contains a non-finite value"
220 )));
221 }
222 Ok(())
223 }
224
225 /// Planning half of [`Mnemo::ingest_thread`]: validate the turns, skip
226 /// the ones already archived, and chunk or truncate the rest. Reads
227 /// only; nothing is written.
228 pub(super) fn plan_ingest(
229 &mut self,
230 thread_id: &str,
231 turns: &[ThreadTurn],
232 ) -> Result<IngestPlan> {
233 if thread_id.is_empty() {
234 return Err(MnemoError::Invalid(
235 "ingest_thread: 'thread_id' must not be empty".into(),
236 ));
237 }
238 // FMT-9b.2: v9 chunks fit one packed page with their metadata.
239 let chunk_bytes = if self.is_v9() {
240 V9_TURN_CHUNK_BYTES
241 } else {
242 THREAD_TURN_CHUNK_BYTES
243 };
244 let mut report = IngestReport::default();
245 if turns.is_empty() {
246 return Ok(IngestPlan {
247 pending: Vec::new(),
248 next_seq: 0,
249 report,
250 });
251 }
252
253 // Build the set of turn_ids already in this thread so the
254 // idempotence check is O(1) per incoming turn. First-pass
255 // cost: one decrypt per existing thread record. A future v9
256 // format bump can move `_mnemo.turn_id` into the catalog row
257 // and drop this scan.
258 // R-004.2: the turn ids already archived and the next `_mnemo.seq`
259 // come from the handle caches: one lazy scan per handle, then O(1)
260 // per ingest instead of two full decrypt scans each time.
261 self.ensure_key_caches()?;
262 self.warm_thread(thread_id)?;
263 let cached = self.layers.as_ref().and_then(|l| l.threads.get(thread_id));
264 let existing_turn_ids: std::collections::HashSet<&str> = cached
265 .map(|tc| tc.turns.keys().map(String::as_str).collect())
266 .unwrap_or_default();
267 // A repeat ingest continues the sequence instead of restarting at 0.
268 let next_seq = cached.and_then(|tc| tc.max_seq).map_or(0, |n| n + 1);
269
270 // Determine each incoming turn's final text pieces (post-
271 // chunking / truncation) before any writes, so the caller can
272 // embed the whole batch in one `embed()` call.
273 let mut pending: Vec<PendingTurn> = Vec::new();
274 let now = memory::now_secs();
275 for t in turns {
276 if t.turn_id.is_empty() {
277 return Err(MnemoError::Invalid(
278 "ingest_thread: turn_id must not be empty".into(),
279 ));
280 }
281 if existing_turn_ids.contains(t.turn_id.as_str()) {
282 report.skipped_existing_turns += 1;
283 continue;
284 }
285 let content_bytes = t.content.len();
286 let is_tool = t.role == "tool";
287 let (parts, truncation) =
288 if is_tool && content_bytes > THREAD_TOOL_RESULT_TRUNCATE_BYTES {
289 report.truncated_turns += 1;
290 let hash = {
291 use sha2::{Digest, Sha256};
292 let h = Sha256::digest(t.content.as_bytes());
293 hex_encode(&h)
294 };
295 let truncated = take_utf8_prefix(&t.content, THREAD_TOOL_RESULT_TRUNCATE_BYTES);
296 (vec![truncated], Some((hash, content_bytes as u64)))
297 } else if content_bytes > chunk_bytes {
298 report.chunked_turns += 1;
299 (chunk_utf8(&t.content, chunk_bytes), None)
300 } else {
301 (vec![t.content.clone()], None)
302 };
303 pending.push(PendingTurn {
304 turn_id: t.turn_id.clone(),
305 role: t.role.clone(),
306 created_at: t.timestamp.unwrap_or(now),
307 parts,
308 truncation,
309 });
310 }
311 Ok(IngestPlan {
312 pending,
313 next_seq,
314 report,
315 })
316 }
317
318 /// Write half of [`Mnemo::ingest_thread`]. `vectors` holds one vector
319 /// per text of `plan.texts()`, already checked. `ttl` stamps every
320 /// new record (the `commit_deltas` `ExpireAfter` policy).
321 pub(super) fn apply_ingest(
322 &mut self,
323 thread_id: &str,
324 plan: IngestPlan,
325 vectors: Vec<Vec<f32>>,
326 ttl: Option<i64>,
327 ) -> Result<IngestReport> {
328 let IngestPlan {
329 pending,
330 mut next_seq,
331 mut report,
332 } = plan;
333 // Order: each pending turn's chunks in part order; each turn
334 // gets a single `_mnemo.seq` slot, so `_mnemo.part`
335 // disambiguates chunks of the same turn.
336 let mut vectors = vectors.into_iter();
337 for p in &pending {
338 let this_seq = next_seq;
339 next_seq += 1;
340 for (part_idx, part_text) in p.parts.iter().enumerate() {
341 let vec = vectors.next().ok_or_else(|| {
342 MnemoError::Invalid("ingest_thread: ran out of vectors".into())
343 })?;
344 let mut m =
345 Memory::new(part_text.clone(), MemoryType::Episodic, vec).with_importance(0.5);
346 if let Some(t) = ttl {
347 m = m.with_ttl(t);
348 }
349 m.created_at = p.created_at;
350 m.accessed_at = p.created_at;
351 m.metadata.insert(
352 MNEMO_META_KIND.into(),
353 Value::String(MNEMO_META_KIND_TURN.into()),
354 );
355 m.metadata.insert(
356 MNEMO_META_THREAD.into(),
357 Value::String(thread_id.to_string()),
358 );
359 m.metadata
360 .insert(MNEMO_META_TURN_ID.into(), Value::String(p.turn_id.clone()));
361 m.metadata
362 .insert(MNEMO_META_SEQ.into(), Value::Number(this_seq.into()));
363 m.metadata
364 .insert(MNEMO_META_ROLE.into(), Value::String(p.role.clone()));
365 if p.parts.len() > 1 {
366 m.metadata.insert(
367 MNEMO_META_PART.into(),
368 Value::Number((part_idx as u64).into()),
369 );
370 }
371 if let Some((sha, len)) = &p.truncation {
372 m.metadata
373 .insert(MNEMO_META_SHA256.into(), Value::String(sha.clone()));
374 m.metadata
375 .insert(MNEMO_META_LENGTH.into(), Value::Number((*len).into()));
376 }
377 self.put(m)?;
378 report.added_records += 1;
379 }
380 }
381 Ok(report)
382 }
383
384 /// Export the archive layer for `thread_id` in `_mnemo.seq`
385 /// order. Each entry is the Memory as stored; chunks of a
386 /// multi-part turn come out in `_mnemo.part` order within their
387 /// `_mnemo.seq`. The record `.content` is the text that was
388 /// stored (post-chunking / post-truncation); callers that need
389 /// the pre-truncation original should round-trip via
390 /// `_mnemo.sha256` + `_mnemo.length`.
391 pub fn thread_export(&mut self, thread_id: &str) -> Result<Vec<Memory>> {
392 // R-004.2: the thread's own records, from the handle caches.
393 // FMT-9e: a cold thread's records were just decoded to warm it;
394 // reuse them instead of decoding (and decompressing) twice.
395 let (entries, mut decoded) = self.thread_entries_decoded(thread_id)?;
396 let mut out: Vec<Memory> = Vec::new();
397 for e in &entries {
398 let m = match decoded.remove(&e.id) {
399 Some(m) => self.finish_memory(e, m)?,
400 None => self.read_memory(e)?,
401 };
402 if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
403 continue;
404 }
405 if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
406 != Some(MNEMO_META_KIND_TURN)
407 {
408 continue;
409 }
410 out.push(m);
411 }
412 out.sort_by(|a, b| {
413 let seq_a = a
414 .metadata
415 .get(MNEMO_META_SEQ)
416 .and_then(|v| v.as_u64())
417 .unwrap_or(0);
418 let seq_b = b
419 .metadata
420 .get(MNEMO_META_SEQ)
421 .and_then(|v| v.as_u64())
422 .unwrap_or(0);
423 let part_a = a
424 .metadata
425 .get(MNEMO_META_PART)
426 .and_then(|v| v.as_u64())
427 .unwrap_or(0);
428 let part_b = b
429 .metadata
430 .get(MNEMO_META_PART)
431 .and_then(|v| v.as_u64())
432 .unwrap_or(0);
433 seq_a.cmp(&seq_b).then_with(|| part_a.cmp(&part_b))
434 });
435 Ok(out)
436 }
437
438 /// Export a thread's archive in the **ingest shape** (Phase 12.7d):
439 /// one [`ExportedTurn`] per source turn, in `_mnemo.seq` order, with
440 /// a chunked turn's parts joined back into one `content`. Passing
441 /// the result back to [`Mnemo::ingest_thread`] for the same thread
442 /// skips every turn; for a new thread it re-creates the same records.
443 /// This is what Python `db.thread_export`, MCP `thread_export` and
444 /// `mnemo thread export` return; [`Mnemo::thread_export`] still
445 /// returns the raw records.
446 pub fn thread_export_turns(&mut self, thread_id: &str) -> Result<Vec<ExportedTurn>> {
447 let records = self.thread_export(thread_id)?;
448 let mut out: Vec<ExportedTurn> = Vec::new();
449 let mut last_seq: Option<u64> = None;
450 for m in records {
451 let meta_str = |k: &str| {
452 m.metadata
453 .get(k)
454 .and_then(|v| v.as_str())
455 .unwrap_or("")
456 .to_string()
457 };
458 let seq = m.metadata.get(MNEMO_META_SEQ).and_then(|v| v.as_u64());
459 // Records are sorted by (seq, part), so the parts of one turn
460 // are adjacent and share a seq.
461 if seq.is_some() && seq == last_seq {
462 if let Some(prev) = out.last_mut() {
463 prev.content.push_str(&m.content);
464 continue;
465 }
466 }
467 last_seq = seq;
468 out.push(ExportedTurn {
469 turn_id: meta_str(MNEMO_META_TURN_ID),
470 role: meta_str(MNEMO_META_ROLE),
471 timestamp: m.created_at,
472 sha256: m
473 .metadata
474 .get(MNEMO_META_SHA256)
475 .and_then(|v| v.as_str())
476 .map(String::from),
477 original_length: m.metadata.get(MNEMO_META_LENGTH).and_then(|v| v.as_u64()),
478 content: m.content,
479 });
480 }
481 Ok(out)
482 }
483
484 /// Soft-delete every record in the archive layer of `thread_id`.
485 /// Returns the number of records marked deleted. Permanent
486 /// removal still requires a subsequent `compact_file`, same as
487 /// any `Mnemo::delete`.
488 ///
489 /// Does **not** touch `_mnemo.kind = "essence"` records or other
490 /// non-turn metadata the thread may carry — only `_mnemo.kind =
491 /// "turn"` records matching the thread id.
492 pub fn thread_forget(&mut self, thread_id: &str) -> Result<usize> {
493 if self.read_only {
494 return Err(MnemoError::ReadOnly);
495 }
496 // Collect first, then delete — can't hold catalog borrows
497 // across `self.delete`.
498 let entries = self.thread_entries(thread_id)?;
499 let mut victims: Vec<Ulid> = Vec::new();
500 for e in &entries {
501 let m = self.read_memory(e)?;
502 if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
503 continue;
504 }
505 if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
506 != Some(MNEMO_META_KIND_TURN)
507 {
508 continue;
509 }
510 victims.push(m.id);
511 }
512 let n = victims.len();
513 for id in &victims {
514 self.delete(id)?;
515 }
516 Ok(n)
517 }
518
519 // --- Phase 12.7b: essence (timeline + sections) -----------------
520}