mnemo_engine/store/threads.rs
1//! Phase 12.7a: the thread archive (ingest, export, forget).
2
3use super::*;
4
5/// One incoming turn after chunking or truncation, ready to embed and
6/// write (Phase 12.7a, split out for 12.7d's batch embed).
7pub(super) struct PendingTurn {
8 pub(super) turn_id: String,
9 pub(super) role: String,
10 pub(super) created_at: i64,
11 pub(super) parts: Vec<String>,
12 /// Hex sha256 + original length, set only for a truncated
13 /// tool-result turn. Applied to the single chunk the truncation
14 /// yielded.
15 pub(super) truncation: Option<(String, u64)>,
16 /// v0.16.1: only markers ([`crate::transcript::is_marker_only`]): kept,
17 /// with the sentinel vector, not embedded.
18 pub(super) markers_only: bool,
19}
20
21/// Output of `Mnemo::plan_ingest`: the turns to write, the first free
22/// `_mnemo.seq`, and the counts gathered so far (skips, chunking,
23/// truncation).
24pub(super) struct IngestPlan {
25 pub(super) pending: Vec<PendingTurn>,
26 pub(super) next_seq: u64,
27 pub(super) report: IngestReport,
28}
29
30impl IngestPlan {
31 /// Every text to embed, in write order (turns, then parts). A
32 /// marker-only turn has none.
33 pub(super) fn texts(&self) -> Vec<&str> {
34 self.pending
35 .iter()
36 .filter(|p| !p.markers_only)
37 .flat_map(|p| p.parts.iter().map(String::as_str))
38 .collect()
39 }
40
41 pub(super) fn text_count(&self) -> usize {
42 self.pending
43 .iter()
44 .filter(|p| !p.markers_only)
45 .map(|p| p.parts.len())
46 .sum()
47 }
48}
49
50/// Hex-encode a byte slice. Low-dependency alternative to pulling in a
51/// hex crate for the lone call site in `ingest_thread`'s SHA-256
52/// recording.
53fn hex_encode(bytes: &[u8]) -> String {
54 const HEX: &[u8; 16] = b"0123456789abcdef";
55 let mut out = String::with_capacity(bytes.len() * 2);
56 for b in bytes {
57 out.push(HEX[(b >> 4) as usize] as char);
58 out.push(HEX[(b & 0x0f) as usize] as char);
59 }
60 out
61}
62
63/// Return the longest UTF-8 prefix of `s` whose byte length is `<=
64/// max_bytes`. Always lands on a `char` boundary.
65fn take_utf8_prefix(s: &str, max_bytes: usize) -> String {
66 if s.len() <= max_bytes {
67 return s.to_string();
68 }
69 // Walk forward through char indices; the last index whose byte
70 // offset fits inside `max_bytes` tells us where to cut.
71 let mut cut = 0usize;
72 for (idx, _) in s.char_indices() {
73 if idx > max_bytes {
74 break;
75 }
76 cut = idx;
77 }
78 s[..cut].to_string()
79}
80
81fn chunk_utf8(s: &str, max_bytes: usize) -> Vec<String> {
82 if s.len() <= max_bytes {
83 return vec![s.to_string()];
84 }
85 let mut out = Vec::new();
86 let mut rest = s;
87 while rest.len() > max_bytes {
88 let head = take_utf8_prefix(rest, max_bytes);
89 let head_bytes = head.len();
90 out.push(head);
91 rest = &rest[head_bytes..];
92 }
93 if !rest.is_empty() {
94 out.push(rest.to_string());
95 }
96 out
97}
98
99impl Mnemo {
100 /// Ingest a batch of turns into the archive layer of `thread_id`.
101 ///
102 /// Each [`ThreadTurn`] becomes one or more `Episodic` records
103 /// tagged with the usual thread-archive reserved-metadata set
104 /// (`_mnemo.kind = "turn"`, `_mnemo.thread`, `_mnemo.turn_id`,
105 /// `_mnemo.seq`, `_mnemo.role`). The ingest path is
106 /// **idempotent** per `turn_id`: a turn whose `turn_id` already
107 /// appears on a record in this thread is skipped, so re-running
108 /// `ingest_thread` with an extended transcript appends only
109 /// what's new.
110 ///
111 /// **Chunking.** A non-tool turn whose UTF-8 content exceeds
112 /// [`THREAD_TURN_CHUNK_BYTES`] (8 KB) is split into chunks at
113 /// char boundaries, each stored as its own record with
114 /// `_mnemo.part = 0, 1, …` under the same `turn_id`.
115 ///
116 /// **Truncation.** A `role = "tool"` turn whose content exceeds
117 /// [`THREAD_TOOL_RESULT_TRUNCATE_BYTES`] (16 KB) is stored
118 /// truncated — the first 16 KB of UTF-8 goes into the record,
119 /// and `_mnemo.sha256` + `_mnemo.length` capture the original
120 /// hash and byte length so the full content can be recovered
121 /// (or verified) out-of-band.
122 ///
123 /// **Embedding.** An embedder must be registered
124 /// ([`Mnemo::set_embedder`]); the content strings (one per
125 /// resulting chunk / truncated record) are batched into a single
126 /// `embed()` call. A turn of nothing but transcript markers
127 /// ([`MNEMO_META_MARKERS_ONLY`], v0.16.1) isn't embedded: it's kept
128 /// with the sentinel vector, and `recall` never returns it.
129 ///
130 /// Call `self.flush()` after a successful ingest to persist.
131 pub fn ingest_thread(&mut self, thread_id: &str, turns: &[ThreadTurn]) -> Result<IngestReport> {
132 if self.read_only {
133 return Err(MnemoError::ReadOnly);
134 }
135 let plan = self.plan_ingest(thread_id, turns)?;
136 if plan.pending.is_empty() {
137 return Ok(plan.report);
138 }
139 // Batched embed: one round trip to the embedder for the whole
140 // ingest, whatever the turn count.
141 let texts = plan.texts();
142 let vectors = if texts.is_empty() {
143 Vec::new()
144 } else {
145 let embedder = self.embedder.as_ref().ok_or_else(|| {
146 MnemoError::Invalid(
147 "ingest_thread: no embedder registered on this handle; call set_embedder first"
148 .into(),
149 )
150 })?;
151 embedder.embed(&texts)?
152 };
153 self.check_vectors("ingest_thread", &vectors, plan.text_count())?;
154 self.apply_ingest(thread_id, plan, vectors, None)
155 }
156
157 /// v0.16.1: how many of `turns` aren't in `thread_id` yet, and would be
158 /// written by an ingest. Unlike [`Mnemo::ingest_thread_texts`], it
159 /// counts a marker-only turn, which has no text to embed. Writes
160 /// nothing.
161 pub fn ingest_thread_new_turns(
162 &mut self,
163 thread_id: &str,
164 turns: &[ThreadTurn],
165 ) -> Result<usize> {
166 Ok(self.plan_ingest(thread_id, turns)?.pending.len())
167 }
168
169 /// MOBILE-1a: the texts [`Mnemo::ingest_thread`] would embed for
170 /// `turns`, in order (one per new chunk or truncated record; turns
171 /// already in the thread are skipped). With
172 /// [`Mnemo::ingest_thread_with_vectors`] it splits an ingest in two, so
173 /// a caller can run its embedder between the calls without holding
174 /// anything the handle needs, as the UniFFI binding does. Empty when
175 /// nothing is new. Writes nothing.
176 pub fn ingest_thread_texts(
177 &mut self,
178 thread_id: &str,
179 turns: &[ThreadTurn],
180 ) -> Result<Vec<String>> {
181 let plan = self.plan_ingest(thread_id, turns)?;
182 Ok(plan.texts().into_iter().map(str::to_string).collect())
183 }
184
185 /// MOBILE-1a: [`Mnemo::ingest_thread`] with caller-supplied vectors,
186 /// one per text [`Mnemo::ingest_thread_texts`] returned for the same
187 /// `thread_id` and `turns`, in the same order. No embedder is needed.
188 /// The ingest is planned again here; if the thread changed in between
189 /// (so the count no longer matches) it returns
190 /// [`MnemoError::Invalid`] and writes nothing, and the caller can ask
191 /// for the texts again.
192 pub fn ingest_thread_with_vectors(
193 &mut self,
194 thread_id: &str,
195 turns: &[ThreadTurn],
196 vectors: &[Vec<f32>],
197 ) -> Result<IngestReport> {
198 if self.read_only {
199 return Err(MnemoError::ReadOnly);
200 }
201 let plan = self.plan_ingest(thread_id, turns)?;
202 if plan.pending.is_empty() && vectors.is_empty() {
203 return Ok(plan.report);
204 }
205 self.check_vectors("ingest_thread_with_vectors", vectors, plan.text_count())?;
206 self.apply_ingest(thread_id, plan, vectors.to_vec(), None)
207 }
208
209 /// Check a batch of embedder output before anything is written:
210 /// one vector per input, the file's width, finite and non-zero.
211 pub(super) fn check_vectors(
212 &self,
213 ctx: &str,
214 vectors: &[Vec<f32>],
215 expected: usize,
216 ) -> Result<()> {
217 if vectors.len() != expected {
218 return Err(MnemoError::Invalid(format!(
219 "{ctx}: embedder returned {} vectors for {expected} texts",
220 vectors.len()
221 )));
222 }
223 for v in vectors {
224 self.check_vector(ctx, v)?;
225 }
226 Ok(())
227 }
228
229 /// The checks `put` applies to one vector, run up front so a batch
230 /// can't fail half-way through its writes.
231 pub(super) fn check_vector(&self, ctx: &str, v: &[f32]) -> Result<()> {
232 if v.len() != self.dimensions {
233 return Err(MnemoError::DimensionMismatch {
234 expected: self.dimensions,
235 got: v.len(),
236 });
237 }
238 if embed::is_zero_vector(v) {
239 return Err(MnemoError::Invalid(format!(
240 "{ctx}: all-zero vector refused"
241 )));
242 }
243 if v.iter().any(|x| !x.is_finite()) {
244 return Err(MnemoError::Invalid(format!(
245 "{ctx}: vector contains a non-finite value"
246 )));
247 }
248 Ok(())
249 }
250
251 /// Planning half of [`Mnemo::ingest_thread`]: validate the turns, skip
252 /// the ones already archived, and chunk or truncate the rest. Reads
253 /// only; nothing is written.
254 pub(super) fn plan_ingest(
255 &mut self,
256 thread_id: &str,
257 turns: &[ThreadTurn],
258 ) -> Result<IngestPlan> {
259 if thread_id.is_empty() {
260 return Err(MnemoError::Invalid(
261 "ingest_thread: 'thread_id' must not be empty".into(),
262 ));
263 }
264 // FMT-9b.2: v9 chunks fit one packed page with their metadata.
265 let chunk_bytes = if self.is_v9() {
266 V9_TURN_CHUNK_BYTES
267 } else {
268 THREAD_TURN_CHUNK_BYTES
269 };
270 let mut report = IngestReport::default();
271 if turns.is_empty() {
272 return Ok(IngestPlan {
273 pending: Vec::new(),
274 next_seq: 0,
275 report,
276 });
277 }
278
279 // Build the set of turn_ids already in this thread so the
280 // idempotence check is O(1) per incoming turn. First-pass
281 // cost: one decrypt per existing thread record. A future v9
282 // format bump can move `_mnemo.turn_id` into the catalog row
283 // and drop this scan.
284 // R-004.2: the turn ids already archived and the next `_mnemo.seq`
285 // come from the handle caches: one lazy scan per handle, then O(1)
286 // per ingest instead of two full decrypt scans each time.
287 self.ensure_key_caches()?;
288 self.warm_thread(thread_id)?;
289 let cached = self.layers.as_ref().and_then(|l| l.threads.get(thread_id));
290 let existing_turn_ids: std::collections::HashSet<&str> = cached
291 .map(|tc| tc.turns.keys().map(String::as_str).collect())
292 .unwrap_or_default();
293 // A repeat ingest continues the sequence instead of restarting at 0.
294 let next_seq = cached.and_then(|tc| tc.max_seq).map_or(0, |n| n + 1);
295
296 // Determine each incoming turn's final text pieces (post-
297 // chunking / truncation) before any writes, so the caller can
298 // embed the whole batch in one `embed()` call.
299 let mut pending: Vec<PendingTurn> = Vec::new();
300 let now = memory::now_secs();
301 for t in turns {
302 if t.turn_id.is_empty() {
303 return Err(MnemoError::Invalid(
304 "ingest_thread: turn_id must not be empty".into(),
305 ));
306 }
307 if existing_turn_ids.contains(t.turn_id.as_str()) {
308 report.skipped_existing_turns += 1;
309 continue;
310 }
311 let content_bytes = t.content.len();
312 let is_tool = t.role == "tool";
313 let (parts, truncation) =
314 if is_tool && content_bytes > THREAD_TOOL_RESULT_TRUNCATE_BYTES {
315 report.truncated_turns += 1;
316 let hash = {
317 use sha2::{Digest, Sha256};
318 let h = Sha256::digest(t.content.as_bytes());
319 hex_encode(&h)
320 };
321 let truncated = take_utf8_prefix(&t.content, THREAD_TOOL_RESULT_TRUNCATE_BYTES);
322 (vec![truncated], Some((hash, content_bytes as u64)))
323 } else if content_bytes > chunk_bytes {
324 report.chunked_turns += 1;
325 (chunk_utf8(&t.content, chunk_bytes), None)
326 } else {
327 (vec![t.content.clone()], None)
328 };
329 pending.push(PendingTurn {
330 turn_id: t.turn_id.clone(),
331 role: t.role.clone(),
332 created_at: t.timestamp.unwrap_or(now),
333 markers_only: crate::transcript::is_marker_only(&t.content),
334 parts,
335 truncation,
336 });
337 }
338 Ok(IngestPlan {
339 pending,
340 next_seq,
341 report,
342 })
343 }
344
345 /// Write half of [`Mnemo::ingest_thread`]. `vectors` holds one vector
346 /// per text of `plan.texts()`, already checked. `ttl` stamps every
347 /// new record (the `commit_deltas` `ExpireAfter` policy).
348 pub(super) fn apply_ingest(
349 &mut self,
350 thread_id: &str,
351 plan: IngestPlan,
352 vectors: Vec<Vec<f32>>,
353 ttl: Option<i64>,
354 ) -> Result<IngestReport> {
355 let IngestPlan {
356 pending,
357 mut next_seq,
358 mut report,
359 } = plan;
360 // Order: each pending turn's chunks in part order; each turn
361 // gets a single `_mnemo.seq` slot, so `_mnemo.part`
362 // disambiguates chunks of the same turn.
363 let mut vectors = vectors.into_iter();
364 let sentinel = self.sentinel_vector();
365 for p in &pending {
366 let this_seq = next_seq;
367 next_seq += 1;
368 for (part_idx, part_text) in p.parts.iter().enumerate() {
369 let vec = if p.markers_only {
370 sentinel.clone()
371 } else {
372 vectors.next().ok_or_else(|| {
373 MnemoError::Invalid("ingest_thread: ran out of vectors".into())
374 })?
375 };
376 let mut m =
377 Memory::new(part_text.clone(), MemoryType::Episodic, vec).with_importance(0.5);
378 if let Some(t) = ttl {
379 m = m.with_ttl(t);
380 }
381 m.created_at = p.created_at;
382 m.accessed_at = p.created_at;
383 m.metadata.insert(
384 MNEMO_META_KIND.into(),
385 Value::String(MNEMO_META_KIND_TURN.into()),
386 );
387 m.metadata.insert(
388 MNEMO_META_THREAD.into(),
389 Value::String(thread_id.to_string()),
390 );
391 m.metadata
392 .insert(MNEMO_META_TURN_ID.into(), Value::String(p.turn_id.clone()));
393 m.metadata
394 .insert(MNEMO_META_SEQ.into(), Value::Number(this_seq.into()));
395 m.metadata
396 .insert(MNEMO_META_ROLE.into(), Value::String(p.role.clone()));
397 if p.parts.len() > 1 {
398 m.metadata.insert(
399 MNEMO_META_PART.into(),
400 Value::Number((part_idx as u64).into()),
401 );
402 }
403 if p.markers_only {
404 m.metadata
405 .insert(MNEMO_META_MARKERS_ONLY.into(), Value::Bool(true));
406 }
407 if let Some((sha, len)) = &p.truncation {
408 m.metadata
409 .insert(MNEMO_META_SHA256.into(), Value::String(sha.clone()));
410 m.metadata
411 .insert(MNEMO_META_LENGTH.into(), Value::Number((*len).into()));
412 }
413 self.put(m)?;
414 report.added_records += 1;
415 }
416 }
417 Ok(report)
418 }
419
420 /// Export the archive layer for `thread_id` in `_mnemo.seq`
421 /// order. Each entry is the Memory as stored; chunks of a
422 /// multi-part turn come out in `_mnemo.part` order within their
423 /// `_mnemo.seq`. The record `.content` is the text that was
424 /// stored (post-chunking / post-truncation); callers that need
425 /// the pre-truncation original should round-trip via
426 /// `_mnemo.sha256` + `_mnemo.length`.
427 pub fn thread_export(&mut self, thread_id: &str) -> Result<Vec<Memory>> {
428 // R-004.2: the thread's own records, from the handle caches.
429 // FMT-9e: a cold thread's records were just decoded to warm it;
430 // reuse them instead of decoding (and decompressing) twice.
431 let (entries, mut decoded) = self.thread_entries_decoded(thread_id)?;
432 let mut out: Vec<Memory> = Vec::new();
433 for e in &entries {
434 let m = match decoded.remove(&e.id) {
435 Some(m) => self.finish_memory(e, m)?,
436 None => self.read_memory(e)?,
437 };
438 if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
439 continue;
440 }
441 if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
442 != Some(MNEMO_META_KIND_TURN)
443 {
444 continue;
445 }
446 out.push(m);
447 }
448 out.sort_by(|a, b| {
449 let seq_a = a
450 .metadata
451 .get(MNEMO_META_SEQ)
452 .and_then(|v| v.as_u64())
453 .unwrap_or(0);
454 let seq_b = b
455 .metadata
456 .get(MNEMO_META_SEQ)
457 .and_then(|v| v.as_u64())
458 .unwrap_or(0);
459 let part_a = a
460 .metadata
461 .get(MNEMO_META_PART)
462 .and_then(|v| v.as_u64())
463 .unwrap_or(0);
464 let part_b = b
465 .metadata
466 .get(MNEMO_META_PART)
467 .and_then(|v| v.as_u64())
468 .unwrap_or(0);
469 seq_a.cmp(&seq_b).then_with(|| part_a.cmp(&part_b))
470 });
471 Ok(out)
472 }
473
474 /// Export a thread's archive in the **ingest shape** (Phase 12.7d):
475 /// one [`ExportedTurn`] per source turn, in `_mnemo.seq` order, with
476 /// a chunked turn's parts joined back into one `content`. Passing
477 /// the result back to [`Mnemo::ingest_thread`] for the same thread
478 /// skips every turn; for a new thread it re-creates the same records.
479 /// This is what Python `db.thread_export`, MCP `thread_export` and
480 /// `mnemo thread export` return; [`Mnemo::thread_export`] still
481 /// returns the raw records.
482 pub fn thread_export_turns(&mut self, thread_id: &str) -> Result<Vec<ExportedTurn>> {
483 let records = self.thread_export(thread_id)?;
484 let mut out: Vec<ExportedTurn> = Vec::new();
485 let mut last_seq: Option<u64> = None;
486 for m in records {
487 let meta_str = |k: &str| {
488 m.metadata
489 .get(k)
490 .and_then(|v| v.as_str())
491 .unwrap_or("")
492 .to_string()
493 };
494 let seq = m.metadata.get(MNEMO_META_SEQ).and_then(|v| v.as_u64());
495 // Records are sorted by (seq, part), so the parts of one turn
496 // are adjacent and share a seq.
497 if seq.is_some() && seq == last_seq {
498 if let Some(prev) = out.last_mut() {
499 prev.content.push_str(&m.content);
500 continue;
501 }
502 }
503 last_seq = seq;
504 out.push(ExportedTurn {
505 turn_id: meta_str(MNEMO_META_TURN_ID),
506 role: meta_str(MNEMO_META_ROLE),
507 timestamp: m.created_at,
508 sha256: m
509 .metadata
510 .get(MNEMO_META_SHA256)
511 .and_then(|v| v.as_str())
512 .map(String::from),
513 original_length: m.metadata.get(MNEMO_META_LENGTH).and_then(|v| v.as_u64()),
514 content: m.content,
515 });
516 }
517 Ok(out)
518 }
519
520 /// Soft-delete every record in the archive layer of `thread_id`.
521 /// Returns the number of records marked deleted. Permanent
522 /// removal still requires a subsequent `compact_file`, same as
523 /// any `Mnemo::delete`.
524 ///
525 /// Does **not** touch `_mnemo.kind = "essence"` records or other
526 /// non-turn metadata the thread may carry — only `_mnemo.kind =
527 /// "turn"` records matching the thread id.
528 pub fn thread_forget(&mut self, thread_id: &str) -> Result<usize> {
529 if self.read_only {
530 return Err(MnemoError::ReadOnly);
531 }
532 // Collect first, then delete — can't hold catalog borrows
533 // across `self.delete`.
534 let entries = self.thread_entries(thread_id)?;
535 let mut victims: Vec<Ulid> = Vec::new();
536 for e in &entries {
537 let m = self.read_memory(e)?;
538 if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
539 continue;
540 }
541 if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
542 != Some(MNEMO_META_KIND_TURN)
543 {
544 continue;
545 }
546 victims.push(m.id);
547 }
548 let n = victims.len();
549 for id in &victims {
550 self.delete(id)?;
551 }
552 Ok(n)
553 }
554
555 // --- Phase 12.7b: essence (timeline + sections) -----------------
556}