Skip to main content

mnemo_engine/store/
threads.rs

1//! Phase 12.7a: the thread archive (ingest, export, forget).
2
3use super::*;
4
5/// One incoming turn after chunking or truncation, ready to embed and
6/// write (Phase 12.7a, split out for 12.7d's batch embed).
7pub(super) struct PendingTurn {
8    pub(super) turn_id: String,
9    pub(super) role: String,
10    pub(super) created_at: i64,
11    pub(super) parts: Vec<String>,
12    /// Hex sha256 + original length, set only for a truncated
13    /// tool-result turn. Applied to the single chunk the truncation
14    /// yielded.
15    pub(super) truncation: Option<(String, u64)>,
16}
17
18/// Output of `Mnemo::plan_ingest`: the turns to write, the first free
19/// `_mnemo.seq`, and the counts gathered so far (skips, chunking,
20/// truncation).
21pub(super) struct IngestPlan {
22    pub(super) pending: Vec<PendingTurn>,
23    pub(super) next_seq: u64,
24    pub(super) report: IngestReport,
25}
26
27impl IngestPlan {
28    /// Every text to embed, in write order (turns, then parts).
29    pub(super) fn texts(&self) -> Vec<&str> {
30        self.pending
31            .iter()
32            .flat_map(|p| p.parts.iter().map(String::as_str))
33            .collect()
34    }
35
36    pub(super) fn text_count(&self) -> usize {
37        self.pending.iter().map(|p| p.parts.len()).sum()
38    }
39}
40
41/// Hex-encode a byte slice. Low-dependency alternative to pulling in a
42/// hex crate for the lone call site in `ingest_thread`'s SHA-256
43/// recording.
44fn hex_encode(bytes: &[u8]) -> String {
45    const HEX: &[u8; 16] = b"0123456789abcdef";
46    let mut out = String::with_capacity(bytes.len() * 2);
47    for b in bytes {
48        out.push(HEX[(b >> 4) as usize] as char);
49        out.push(HEX[(b & 0x0f) as usize] as char);
50    }
51    out
52}
53
54/// Return the longest UTF-8 prefix of `s` whose byte length is `<=
55/// max_bytes`. Always lands on a `char` boundary.
56fn take_utf8_prefix(s: &str, max_bytes: usize) -> String {
57    if s.len() <= max_bytes {
58        return s.to_string();
59    }
60    // Walk forward through char indices; the last index whose byte
61    // offset fits inside `max_bytes` tells us where to cut.
62    let mut cut = 0usize;
63    for (idx, _) in s.char_indices() {
64        if idx > max_bytes {
65            break;
66        }
67        cut = idx;
68    }
69    s[..cut].to_string()
70}
71
72fn chunk_utf8(s: &str, max_bytes: usize) -> Vec<String> {
73    if s.len() <= max_bytes {
74        return vec![s.to_string()];
75    }
76    let mut out = Vec::new();
77    let mut rest = s;
78    while rest.len() > max_bytes {
79        let head = take_utf8_prefix(rest, max_bytes);
80        let head_bytes = head.len();
81        out.push(head);
82        rest = &rest[head_bytes..];
83    }
84    if !rest.is_empty() {
85        out.push(rest.to_string());
86    }
87    out
88}
89
90impl Mnemo {
91    /// Ingest a batch of turns into the archive layer of `thread_id`.
92    ///
93    /// Each [`ThreadTurn`] becomes one or more `Episodic` records
94    /// tagged with the usual thread-archive reserved-metadata set
95    /// (`_mnemo.kind = "turn"`, `_mnemo.thread`, `_mnemo.turn_id`,
96    /// `_mnemo.seq`, `_mnemo.role`). The ingest path is
97    /// **idempotent** per `turn_id`: a turn whose `turn_id` already
98    /// appears on a record in this thread is skipped, so re-running
99    /// `ingest_thread` with an extended transcript appends only
100    /// what's new.
101    ///
102    /// **Chunking.** A non-tool turn whose UTF-8 content exceeds
103    /// [`THREAD_TURN_CHUNK_BYTES`] (8 KB) is split into chunks at
104    /// char boundaries, each stored as its own record with
105    /// `_mnemo.part = 0, 1, …` under the same `turn_id`.
106    ///
107    /// **Truncation.** A `role = "tool"` turn whose content exceeds
108    /// [`THREAD_TOOL_RESULT_TRUNCATE_BYTES`] (16 KB) is stored
109    /// truncated — the first 16 KB of UTF-8 goes into the record,
110    /// and `_mnemo.sha256` + `_mnemo.length` capture the original
111    /// hash and byte length so the full content can be recovered
112    /// (or verified) out-of-band.
113    ///
114    /// **Embedding.** An embedder must be registered
115    /// ([`Mnemo::set_embedder`]); the content strings (one per
116    /// resulting chunk / truncated record) are batched into a single
117    /// `embed()` call.
118    ///
119    /// Call `self.flush()` after a successful ingest to persist.
120    pub fn ingest_thread(&mut self, thread_id: &str, turns: &[ThreadTurn]) -> Result<IngestReport> {
121        if self.read_only {
122            return Err(MnemoError::ReadOnly);
123        }
124        let plan = self.plan_ingest(thread_id, turns)?;
125        if plan.pending.is_empty() {
126            return Ok(plan.report);
127        }
128        // Batched embed: one round trip to the embedder for the whole
129        // ingest, whatever the turn count.
130        let vectors = {
131            let embedder = self.embedder.as_ref().ok_or_else(|| {
132                MnemoError::Invalid(
133                    "ingest_thread: no embedder registered on this handle; call set_embedder first"
134                        .into(),
135                )
136            })?;
137            embedder.embed(&plan.texts())?
138        };
139        self.check_vectors("ingest_thread", &vectors, plan.text_count())?;
140        self.apply_ingest(thread_id, plan, vectors, None)
141    }
142
143    /// MOBILE-1a: the texts [`Mnemo::ingest_thread`] would embed for
144    /// `turns`, in order (one per new chunk or truncated record; turns
145    /// already in the thread are skipped). With
146    /// [`Mnemo::ingest_thread_with_vectors`] it splits an ingest in two, so
147    /// a caller can run its embedder between the calls without holding
148    /// anything the handle needs, as the UniFFI binding does. Empty when
149    /// nothing is new. Writes nothing.
150    pub fn ingest_thread_texts(
151        &mut self,
152        thread_id: &str,
153        turns: &[ThreadTurn],
154    ) -> Result<Vec<String>> {
155        let plan = self.plan_ingest(thread_id, turns)?;
156        Ok(plan.texts().into_iter().map(str::to_string).collect())
157    }
158
159    /// MOBILE-1a: [`Mnemo::ingest_thread`] with caller-supplied vectors,
160    /// one per text [`Mnemo::ingest_thread_texts`] returned for the same
161    /// `thread_id` and `turns`, in the same order. No embedder is needed.
162    /// The ingest is planned again here; if the thread changed in between
163    /// (so the count no longer matches) it returns
164    /// [`MnemoError::Invalid`] and writes nothing, and the caller can ask
165    /// for the texts again.
166    pub fn ingest_thread_with_vectors(
167        &mut self,
168        thread_id: &str,
169        turns: &[ThreadTurn],
170        vectors: &[Vec<f32>],
171    ) -> Result<IngestReport> {
172        if self.read_only {
173            return Err(MnemoError::ReadOnly);
174        }
175        let plan = self.plan_ingest(thread_id, turns)?;
176        if plan.pending.is_empty() && vectors.is_empty() {
177            return Ok(plan.report);
178        }
179        self.check_vectors("ingest_thread_with_vectors", vectors, plan.text_count())?;
180        self.apply_ingest(thread_id, plan, vectors.to_vec(), None)
181    }
182
183    /// Check a batch of embedder output before anything is written:
184    /// one vector per input, the file's width, finite and non-zero.
185    pub(super) fn check_vectors(
186        &self,
187        ctx: &str,
188        vectors: &[Vec<f32>],
189        expected: usize,
190    ) -> Result<()> {
191        if vectors.len() != expected {
192            return Err(MnemoError::Invalid(format!(
193                "{ctx}: embedder returned {} vectors for {expected} texts",
194                vectors.len()
195            )));
196        }
197        for v in vectors {
198            self.check_vector(ctx, v)?;
199        }
200        Ok(())
201    }
202
203    /// The checks `put` applies to one vector, run up front so a batch
204    /// can't fail half-way through its writes.
205    pub(super) fn check_vector(&self, ctx: &str, v: &[f32]) -> Result<()> {
206        if v.len() != self.dimensions {
207            return Err(MnemoError::DimensionMismatch {
208                expected: self.dimensions,
209                got: v.len(),
210            });
211        }
212        if embed::is_zero_vector(v) {
213            return Err(MnemoError::Invalid(format!(
214                "{ctx}: all-zero vector refused"
215            )));
216        }
217        if v.iter().any(|x| !x.is_finite()) {
218            return Err(MnemoError::Invalid(format!(
219                "{ctx}: vector contains a non-finite value"
220            )));
221        }
222        Ok(())
223    }
224
225    /// Planning half of [`Mnemo::ingest_thread`]: validate the turns, skip
226    /// the ones already archived, and chunk or truncate the rest. Reads
227    /// only; nothing is written.
228    pub(super) fn plan_ingest(
229        &mut self,
230        thread_id: &str,
231        turns: &[ThreadTurn],
232    ) -> Result<IngestPlan> {
233        if thread_id.is_empty() {
234            return Err(MnemoError::Invalid(
235                "ingest_thread: 'thread_id' must not be empty".into(),
236            ));
237        }
238        // FMT-9b.2: v9 chunks fit one packed page with their metadata.
239        let chunk_bytes = if self.is_v9() {
240            V9_TURN_CHUNK_BYTES
241        } else {
242            THREAD_TURN_CHUNK_BYTES
243        };
244        let mut report = IngestReport::default();
245        if turns.is_empty() {
246            return Ok(IngestPlan {
247                pending: Vec::new(),
248                next_seq: 0,
249                report,
250            });
251        }
252
253        // Build the set of turn_ids already in this thread so the
254        // idempotence check is O(1) per incoming turn. First-pass
255        // cost: one decrypt per existing thread record. A future v9
256        // format bump can move `_mnemo.turn_id` into the catalog row
257        // and drop this scan.
258        // R-004.2: the turn ids already archived and the next `_mnemo.seq`
259        // come from the handle caches: one lazy scan per handle, then O(1)
260        // per ingest instead of two full decrypt scans each time.
261        self.ensure_key_caches()?;
262        self.warm_thread(thread_id)?;
263        let cached = self.layers.as_ref().and_then(|l| l.threads.get(thread_id));
264        let existing_turn_ids: std::collections::HashSet<&str> = cached
265            .map(|tc| tc.turns.keys().map(String::as_str).collect())
266            .unwrap_or_default();
267        // A repeat ingest continues the sequence instead of restarting at 0.
268        let next_seq = cached.and_then(|tc| tc.max_seq).map_or(0, |n| n + 1);
269
270        // Determine each incoming turn's final text pieces (post-
271        // chunking / truncation) before any writes, so the caller can
272        // embed the whole batch in one `embed()` call.
273        let mut pending: Vec<PendingTurn> = Vec::new();
274        let now = memory::now_secs();
275        for t in turns {
276            if t.turn_id.is_empty() {
277                return Err(MnemoError::Invalid(
278                    "ingest_thread: turn_id must not be empty".into(),
279                ));
280            }
281            if existing_turn_ids.contains(t.turn_id.as_str()) {
282                report.skipped_existing_turns += 1;
283                continue;
284            }
285            let content_bytes = t.content.len();
286            let is_tool = t.role == "tool";
287            let (parts, truncation) =
288                if is_tool && content_bytes > THREAD_TOOL_RESULT_TRUNCATE_BYTES {
289                    report.truncated_turns += 1;
290                    let hash = {
291                        use sha2::{Digest, Sha256};
292                        let h = Sha256::digest(t.content.as_bytes());
293                        hex_encode(&h)
294                    };
295                    let truncated = take_utf8_prefix(&t.content, THREAD_TOOL_RESULT_TRUNCATE_BYTES);
296                    (vec![truncated], Some((hash, content_bytes as u64)))
297                } else if content_bytes > chunk_bytes {
298                    report.chunked_turns += 1;
299                    (chunk_utf8(&t.content, chunk_bytes), None)
300                } else {
301                    (vec![t.content.clone()], None)
302                };
303            pending.push(PendingTurn {
304                turn_id: t.turn_id.clone(),
305                role: t.role.clone(),
306                created_at: t.timestamp.unwrap_or(now),
307                parts,
308                truncation,
309            });
310        }
311        Ok(IngestPlan {
312            pending,
313            next_seq,
314            report,
315        })
316    }
317
318    /// Write half of [`Mnemo::ingest_thread`]. `vectors` holds one vector
319    /// per text of `plan.texts()`, already checked. `ttl` stamps every
320    /// new record (the `commit_deltas` `ExpireAfter` policy).
321    pub(super) fn apply_ingest(
322        &mut self,
323        thread_id: &str,
324        plan: IngestPlan,
325        vectors: Vec<Vec<f32>>,
326        ttl: Option<i64>,
327    ) -> Result<IngestReport> {
328        let IngestPlan {
329            pending,
330            mut next_seq,
331            mut report,
332        } = plan;
333        // Order: each pending turn's chunks in part order; each turn
334        // gets a single `_mnemo.seq` slot, so `_mnemo.part`
335        // disambiguates chunks of the same turn.
336        let mut vectors = vectors.into_iter();
337        for p in &pending {
338            let this_seq = next_seq;
339            next_seq += 1;
340            for (part_idx, part_text) in p.parts.iter().enumerate() {
341                let vec = vectors.next().ok_or_else(|| {
342                    MnemoError::Invalid("ingest_thread: ran out of vectors".into())
343                })?;
344                let mut m =
345                    Memory::new(part_text.clone(), MemoryType::Episodic, vec).with_importance(0.5);
346                if let Some(t) = ttl {
347                    m = m.with_ttl(t);
348                }
349                m.created_at = p.created_at;
350                m.accessed_at = p.created_at;
351                m.metadata.insert(
352                    MNEMO_META_KIND.into(),
353                    Value::String(MNEMO_META_KIND_TURN.into()),
354                );
355                m.metadata.insert(
356                    MNEMO_META_THREAD.into(),
357                    Value::String(thread_id.to_string()),
358                );
359                m.metadata
360                    .insert(MNEMO_META_TURN_ID.into(), Value::String(p.turn_id.clone()));
361                m.metadata
362                    .insert(MNEMO_META_SEQ.into(), Value::Number(this_seq.into()));
363                m.metadata
364                    .insert(MNEMO_META_ROLE.into(), Value::String(p.role.clone()));
365                if p.parts.len() > 1 {
366                    m.metadata.insert(
367                        MNEMO_META_PART.into(),
368                        Value::Number((part_idx as u64).into()),
369                    );
370                }
371                if let Some((sha, len)) = &p.truncation {
372                    m.metadata
373                        .insert(MNEMO_META_SHA256.into(), Value::String(sha.clone()));
374                    m.metadata
375                        .insert(MNEMO_META_LENGTH.into(), Value::Number((*len).into()));
376                }
377                self.put(m)?;
378                report.added_records += 1;
379            }
380        }
381        Ok(report)
382    }
383
384    /// Export the archive layer for `thread_id` in `_mnemo.seq`
385    /// order. Each entry is the Memory as stored; chunks of a
386    /// multi-part turn come out in `_mnemo.part` order within their
387    /// `_mnemo.seq`. The record `.content` is the text that was
388    /// stored (post-chunking / post-truncation); callers that need
389    /// the pre-truncation original should round-trip via
390    /// `_mnemo.sha256` + `_mnemo.length`.
391    pub fn thread_export(&mut self, thread_id: &str) -> Result<Vec<Memory>> {
392        // R-004.2: the thread's own records, from the handle caches.
393        // FMT-9e: a cold thread's records were just decoded to warm it;
394        // reuse them instead of decoding (and decompressing) twice.
395        let (entries, mut decoded) = self.thread_entries_decoded(thread_id)?;
396        let mut out: Vec<Memory> = Vec::new();
397        for e in &entries {
398            let m = match decoded.remove(&e.id) {
399                Some(m) => self.finish_memory(e, m)?,
400                None => self.read_memory(e)?,
401            };
402            if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
403                continue;
404            }
405            if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
406                != Some(MNEMO_META_KIND_TURN)
407            {
408                continue;
409            }
410            out.push(m);
411        }
412        out.sort_by(|a, b| {
413            let seq_a = a
414                .metadata
415                .get(MNEMO_META_SEQ)
416                .and_then(|v| v.as_u64())
417                .unwrap_or(0);
418            let seq_b = b
419                .metadata
420                .get(MNEMO_META_SEQ)
421                .and_then(|v| v.as_u64())
422                .unwrap_or(0);
423            let part_a = a
424                .metadata
425                .get(MNEMO_META_PART)
426                .and_then(|v| v.as_u64())
427                .unwrap_or(0);
428            let part_b = b
429                .metadata
430                .get(MNEMO_META_PART)
431                .and_then(|v| v.as_u64())
432                .unwrap_or(0);
433            seq_a.cmp(&seq_b).then_with(|| part_a.cmp(&part_b))
434        });
435        Ok(out)
436    }
437
438    /// Export a thread's archive in the **ingest shape** (Phase 12.7d):
439    /// one [`ExportedTurn`] per source turn, in `_mnemo.seq` order, with
440    /// a chunked turn's parts joined back into one `content`. Passing
441    /// the result back to [`Mnemo::ingest_thread`] for the same thread
442    /// skips every turn; for a new thread it re-creates the same records.
443    /// This is what Python `db.thread_export`, MCP `thread_export` and
444    /// `mnemo thread export` return; [`Mnemo::thread_export`] still
445    /// returns the raw records.
446    pub fn thread_export_turns(&mut self, thread_id: &str) -> Result<Vec<ExportedTurn>> {
447        let records = self.thread_export(thread_id)?;
448        let mut out: Vec<ExportedTurn> = Vec::new();
449        let mut last_seq: Option<u64> = None;
450        for m in records {
451            let meta_str = |k: &str| {
452                m.metadata
453                    .get(k)
454                    .and_then(|v| v.as_str())
455                    .unwrap_or("")
456                    .to_string()
457            };
458            let seq = m.metadata.get(MNEMO_META_SEQ).and_then(|v| v.as_u64());
459            // Records are sorted by (seq, part), so the parts of one turn
460            // are adjacent and share a seq.
461            if seq.is_some() && seq == last_seq {
462                if let Some(prev) = out.last_mut() {
463                    prev.content.push_str(&m.content);
464                    continue;
465                }
466            }
467            last_seq = seq;
468            out.push(ExportedTurn {
469                turn_id: meta_str(MNEMO_META_TURN_ID),
470                role: meta_str(MNEMO_META_ROLE),
471                timestamp: m.created_at,
472                sha256: m
473                    .metadata
474                    .get(MNEMO_META_SHA256)
475                    .and_then(|v| v.as_str())
476                    .map(String::from),
477                original_length: m.metadata.get(MNEMO_META_LENGTH).and_then(|v| v.as_u64()),
478                content: m.content,
479            });
480        }
481        Ok(out)
482    }
483
484    /// Soft-delete every record in the archive layer of `thread_id`.
485    /// Returns the number of records marked deleted. Permanent
486    /// removal still requires a subsequent `compact_file`, same as
487    /// any `Mnemo::delete`.
488    ///
489    /// Does **not** touch `_mnemo.kind = "essence"` records or other
490    /// non-turn metadata the thread may carry — only `_mnemo.kind =
491    /// "turn"` records matching the thread id.
492    pub fn thread_forget(&mut self, thread_id: &str) -> Result<usize> {
493        if self.read_only {
494            return Err(MnemoError::ReadOnly);
495        }
496        // Collect first, then delete — can't hold catalog borrows
497        // across `self.delete`.
498        let entries = self.thread_entries(thread_id)?;
499        let mut victims: Vec<Ulid> = Vec::new();
500        for e in &entries {
501            let m = self.read_memory(e)?;
502            if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
503                continue;
504            }
505            if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
506                != Some(MNEMO_META_KIND_TURN)
507            {
508                continue;
509            }
510            victims.push(m.id);
511        }
512        let n = victims.len();
513        for id in &victims {
514            self.delete(id)?;
515        }
516        Ok(n)
517    }
518
519    // --- Phase 12.7b: essence (timeline + sections) -----------------
520}