Skip to main content

mnemo_engine/store/
threads.rs

1//! Phase 12.7a: the thread archive (ingest, export, forget).
2
3use super::*;
4
5/// One incoming turn after chunking or truncation, ready to embed and
6/// write (Phase 12.7a, split out for 12.7d's batch embed).
7pub(super) struct PendingTurn {
8    pub(super) turn_id: String,
9    pub(super) role: String,
10    pub(super) created_at: i64,
11    pub(super) parts: Vec<String>,
12    /// Hex sha256 + original length, set only for a truncated
13    /// tool-result turn. Applied to the single chunk the truncation
14    /// yielded.
15    pub(super) truncation: Option<(String, u64)>,
16    /// v0.16.1: only markers ([`crate::transcript::is_marker_only`]): kept,
17    /// with the sentinel vector, not embedded.
18    pub(super) markers_only: bool,
19}
20
21/// Output of `Mnemo::plan_ingest`: the turns to write, the first free
22/// `_mnemo.seq`, and the counts gathered so far (skips, chunking,
23/// truncation).
24pub(super) struct IngestPlan {
25    pub(super) pending: Vec<PendingTurn>,
26    pub(super) next_seq: u64,
27    pub(super) report: IngestReport,
28}
29
30impl IngestPlan {
31    /// Every text to embed, in write order (turns, then parts). A
32    /// marker-only turn has none.
33    pub(super) fn texts(&self) -> Vec<&str> {
34        self.pending
35            .iter()
36            .filter(|p| !p.markers_only)
37            .flat_map(|p| p.parts.iter().map(String::as_str))
38            .collect()
39    }
40
41    pub(super) fn text_count(&self) -> usize {
42        self.pending
43            .iter()
44            .filter(|p| !p.markers_only)
45            .map(|p| p.parts.len())
46            .sum()
47    }
48}
49
50/// Hex-encode a byte slice. Low-dependency alternative to pulling in a
51/// hex crate for the lone call site in `ingest_thread`'s SHA-256
52/// recording.
53fn hex_encode(bytes: &[u8]) -> String {
54    const HEX: &[u8; 16] = b"0123456789abcdef";
55    let mut out = String::with_capacity(bytes.len() * 2);
56    for b in bytes {
57        out.push(HEX[(b >> 4) as usize] as char);
58        out.push(HEX[(b & 0x0f) as usize] as char);
59    }
60    out
61}
62
63/// Return the longest UTF-8 prefix of `s` whose byte length is `<=
64/// max_bytes`. Always lands on a `char` boundary.
65fn take_utf8_prefix(s: &str, max_bytes: usize) -> String {
66    if s.len() <= max_bytes {
67        return s.to_string();
68    }
69    // Walk forward through char indices; the last index whose byte
70    // offset fits inside `max_bytes` tells us where to cut.
71    let mut cut = 0usize;
72    for (idx, _) in s.char_indices() {
73        if idx > max_bytes {
74            break;
75        }
76        cut = idx;
77    }
78    s[..cut].to_string()
79}
80
81fn chunk_utf8(s: &str, max_bytes: usize) -> Vec<String> {
82    if s.len() <= max_bytes {
83        return vec![s.to_string()];
84    }
85    let mut out = Vec::new();
86    let mut rest = s;
87    while rest.len() > max_bytes {
88        let head = take_utf8_prefix(rest, max_bytes);
89        let head_bytes = head.len();
90        out.push(head);
91        rest = &rest[head_bytes..];
92    }
93    if !rest.is_empty() {
94        out.push(rest.to_string());
95    }
96    out
97}
98
99impl Mnemo {
100    /// Ingest a batch of turns into the archive layer of `thread_id`.
101    ///
102    /// Each [`ThreadTurn`] becomes one or more `Episodic` records
103    /// tagged with the usual thread-archive reserved-metadata set
104    /// (`_mnemo.kind = "turn"`, `_mnemo.thread`, `_mnemo.turn_id`,
105    /// `_mnemo.seq`, `_mnemo.role`). The ingest path is
106    /// **idempotent** per `turn_id`: a turn whose `turn_id` already
107    /// appears on a record in this thread is skipped, so re-running
108    /// `ingest_thread` with an extended transcript appends only
109    /// what's new.
110    ///
111    /// **Chunking.** A non-tool turn whose UTF-8 content exceeds
112    /// [`THREAD_TURN_CHUNK_BYTES`] (8 KB) is split into chunks at
113    /// char boundaries, each stored as its own record with
114    /// `_mnemo.part = 0, 1, …` under the same `turn_id`.
115    ///
116    /// **Truncation.** A `role = "tool"` turn whose content exceeds
117    /// [`THREAD_TOOL_RESULT_TRUNCATE_BYTES`] (16 KB) is stored
118    /// truncated — the first 16 KB of UTF-8 goes into the record,
119    /// and `_mnemo.sha256` + `_mnemo.length` capture the original
120    /// hash and byte length so the full content can be recovered
121    /// (or verified) out-of-band.
122    ///
123    /// **Embedding.** An embedder must be registered
124    /// ([`Mnemo::set_embedder`]); the content strings (one per
125    /// resulting chunk / truncated record) are batched into a single
126    /// `embed()` call. A turn of nothing but transcript markers
127    /// ([`MNEMO_META_MARKERS_ONLY`], v0.16.1) isn't embedded: it's kept
128    /// with the sentinel vector, and `recall` never returns it.
129    ///
130    /// Call `self.flush()` after a successful ingest to persist.
131    pub fn ingest_thread(&mut self, thread_id: &str, turns: &[ThreadTurn]) -> Result<IngestReport> {
132        if self.read_only {
133            return Err(MnemoError::ReadOnly);
134        }
135        let plan = self.plan_ingest(thread_id, turns)?;
136        if plan.pending.is_empty() {
137            return Ok(plan.report);
138        }
139        // Batched embed: one round trip to the embedder for the whole
140        // ingest, whatever the turn count.
141        let texts = plan.texts();
142        let vectors = if texts.is_empty() {
143            Vec::new()
144        } else {
145            let embedder = self.embedder.as_ref().ok_or_else(|| {
146                MnemoError::Invalid(
147                    "ingest_thread: no embedder registered on this handle; call set_embedder first"
148                        .into(),
149                )
150            })?;
151            embedder.embed(&texts)?
152        };
153        self.check_vectors("ingest_thread", &vectors, plan.text_count())?;
154        self.apply_ingest(thread_id, plan, vectors, None)
155    }
156
157    /// v0.16.1: how many of `turns` aren't in `thread_id` yet, and would be
158    /// written by an ingest. Unlike [`Mnemo::ingest_thread_texts`], it
159    /// counts a marker-only turn, which has no text to embed. Writes
160    /// nothing.
161    pub fn ingest_thread_new_turns(
162        &mut self,
163        thread_id: &str,
164        turns: &[ThreadTurn],
165    ) -> Result<usize> {
166        Ok(self.plan_ingest(thread_id, turns)?.pending.len())
167    }
168
169    /// MOBILE-1a: the texts [`Mnemo::ingest_thread`] would embed for
170    /// `turns`, in order (one per new chunk or truncated record; turns
171    /// already in the thread are skipped). With
172    /// [`Mnemo::ingest_thread_with_vectors`] it splits an ingest in two, so
173    /// a caller can run its embedder between the calls without holding
174    /// anything the handle needs, as the UniFFI binding does. Empty when
175    /// nothing is new. Writes nothing.
176    pub fn ingest_thread_texts(
177        &mut self,
178        thread_id: &str,
179        turns: &[ThreadTurn],
180    ) -> Result<Vec<String>> {
181        let plan = self.plan_ingest(thread_id, turns)?;
182        Ok(plan.texts().into_iter().map(str::to_string).collect())
183    }
184
185    /// MOBILE-1a: [`Mnemo::ingest_thread`] with caller-supplied vectors,
186    /// one per text [`Mnemo::ingest_thread_texts`] returned for the same
187    /// `thread_id` and `turns`, in the same order. No embedder is needed.
188    /// The ingest is planned again here; if the thread changed in between
189    /// (so the count no longer matches) it returns
190    /// [`MnemoError::Invalid`] and writes nothing, and the caller can ask
191    /// for the texts again.
192    pub fn ingest_thread_with_vectors(
193        &mut self,
194        thread_id: &str,
195        turns: &[ThreadTurn],
196        vectors: &[Vec<f32>],
197    ) -> Result<IngestReport> {
198        if self.read_only {
199            return Err(MnemoError::ReadOnly);
200        }
201        let plan = self.plan_ingest(thread_id, turns)?;
202        if plan.pending.is_empty() && vectors.is_empty() {
203            return Ok(plan.report);
204        }
205        self.check_vectors("ingest_thread_with_vectors", vectors, plan.text_count())?;
206        self.apply_ingest(thread_id, plan, vectors.to_vec(), None)
207    }
208
209    /// Check a batch of embedder output before anything is written:
210    /// one vector per input, the file's width, finite and non-zero.
211    pub(super) fn check_vectors(
212        &self,
213        ctx: &str,
214        vectors: &[Vec<f32>],
215        expected: usize,
216    ) -> Result<()> {
217        if vectors.len() != expected {
218            return Err(MnemoError::Invalid(format!(
219                "{ctx}: embedder returned {} vectors for {expected} texts",
220                vectors.len()
221            )));
222        }
223        for v in vectors {
224            self.check_vector(ctx, v)?;
225        }
226        Ok(())
227    }
228
229    /// The checks `put` applies to one vector, run up front so a batch
230    /// can't fail half-way through its writes.
231    pub(super) fn check_vector(&self, ctx: &str, v: &[f32]) -> Result<()> {
232        if v.len() != self.dimensions {
233            return Err(MnemoError::DimensionMismatch {
234                expected: self.dimensions,
235                got: v.len(),
236            });
237        }
238        if embed::is_zero_vector(v) {
239            return Err(MnemoError::Invalid(format!(
240                "{ctx}: all-zero vector refused"
241            )));
242        }
243        if v.iter().any(|x| !x.is_finite()) {
244            return Err(MnemoError::Invalid(format!(
245                "{ctx}: vector contains a non-finite value"
246            )));
247        }
248        Ok(())
249    }
250
251    /// Planning half of [`Mnemo::ingest_thread`]: validate the turns, skip
252    /// the ones already archived, and chunk or truncate the rest. Reads
253    /// only; nothing is written.
254    pub(super) fn plan_ingest(
255        &mut self,
256        thread_id: &str,
257        turns: &[ThreadTurn],
258    ) -> Result<IngestPlan> {
259        if thread_id.is_empty() {
260            return Err(MnemoError::Invalid(
261                "ingest_thread: 'thread_id' must not be empty".into(),
262            ));
263        }
264        // FMT-9b.2: v9 chunks fit one packed page with their metadata.
265        let chunk_bytes = if self.is_v9() {
266            V9_TURN_CHUNK_BYTES
267        } else {
268            THREAD_TURN_CHUNK_BYTES
269        };
270        let mut report = IngestReport::default();
271        if turns.is_empty() {
272            return Ok(IngestPlan {
273                pending: Vec::new(),
274                next_seq: 0,
275                report,
276            });
277        }
278
279        // Build the set of turn_ids already in this thread so the
280        // idempotence check is O(1) per incoming turn. First-pass
281        // cost: one decrypt per existing thread record. A future v9
282        // format bump can move `_mnemo.turn_id` into the catalog row
283        // and drop this scan.
284        // R-004.2: the turn ids already archived and the next `_mnemo.seq`
285        // come from the handle caches: one lazy scan per handle, then O(1)
286        // per ingest instead of two full decrypt scans each time.
287        self.ensure_key_caches()?;
288        self.warm_thread(thread_id)?;
289        let cached = self.layers.as_ref().and_then(|l| l.threads.get(thread_id));
290        let existing_turn_ids: std::collections::HashSet<&str> = cached
291            .map(|tc| tc.turns.keys().map(String::as_str).collect())
292            .unwrap_or_default();
293        // A repeat ingest continues the sequence instead of restarting at 0.
294        let next_seq = cached.and_then(|tc| tc.max_seq).map_or(0, |n| n + 1);
295
296        // Determine each incoming turn's final text pieces (post-
297        // chunking / truncation) before any writes, so the caller can
298        // embed the whole batch in one `embed()` call.
299        let mut pending: Vec<PendingTurn> = Vec::new();
300        let now = memory::now_secs();
301        for t in turns {
302            if t.turn_id.is_empty() {
303                return Err(MnemoError::Invalid(
304                    "ingest_thread: turn_id must not be empty".into(),
305                ));
306            }
307            if existing_turn_ids.contains(t.turn_id.as_str()) {
308                report.skipped_existing_turns += 1;
309                continue;
310            }
311            let content_bytes = t.content.len();
312            let is_tool = t.role == "tool";
313            let (parts, truncation) =
314                if is_tool && content_bytes > THREAD_TOOL_RESULT_TRUNCATE_BYTES {
315                    report.truncated_turns += 1;
316                    let hash = {
317                        use sha2::{Digest, Sha256};
318                        let h = Sha256::digest(t.content.as_bytes());
319                        hex_encode(&h)
320                    };
321                    let truncated = take_utf8_prefix(&t.content, THREAD_TOOL_RESULT_TRUNCATE_BYTES);
322                    (vec![truncated], Some((hash, content_bytes as u64)))
323                } else if content_bytes > chunk_bytes {
324                    report.chunked_turns += 1;
325                    (chunk_utf8(&t.content, chunk_bytes), None)
326                } else {
327                    (vec![t.content.clone()], None)
328                };
329            pending.push(PendingTurn {
330                turn_id: t.turn_id.clone(),
331                role: t.role.clone(),
332                created_at: t.timestamp.unwrap_or(now),
333                markers_only: crate::transcript::is_marker_only(&t.content),
334                parts,
335                truncation,
336            });
337        }
338        Ok(IngestPlan {
339            pending,
340            next_seq,
341            report,
342        })
343    }
344
345    /// Write half of [`Mnemo::ingest_thread`]. `vectors` holds one vector
346    /// per text of `plan.texts()`, already checked. `ttl` stamps every
347    /// new record (the `commit_deltas` `ExpireAfter` policy).
348    pub(super) fn apply_ingest(
349        &mut self,
350        thread_id: &str,
351        plan: IngestPlan,
352        vectors: Vec<Vec<f32>>,
353        ttl: Option<i64>,
354    ) -> Result<IngestReport> {
355        let IngestPlan {
356            pending,
357            mut next_seq,
358            mut report,
359        } = plan;
360        // Order: each pending turn's chunks in part order; each turn
361        // gets a single `_mnemo.seq` slot, so `_mnemo.part`
362        // disambiguates chunks of the same turn.
363        let mut vectors = vectors.into_iter();
364        let sentinel = self.sentinel_vector();
365        for p in &pending {
366            let this_seq = next_seq;
367            next_seq += 1;
368            for (part_idx, part_text) in p.parts.iter().enumerate() {
369                let vec = if p.markers_only {
370                    sentinel.clone()
371                } else {
372                    vectors.next().ok_or_else(|| {
373                        MnemoError::Invalid("ingest_thread: ran out of vectors".into())
374                    })?
375                };
376                let mut m =
377                    Memory::new(part_text.clone(), MemoryType::Episodic, vec).with_importance(0.5);
378                if let Some(t) = ttl {
379                    m = m.with_ttl(t);
380                }
381                m.created_at = p.created_at;
382                m.accessed_at = p.created_at;
383                m.metadata.insert(
384                    MNEMO_META_KIND.into(),
385                    Value::String(MNEMO_META_KIND_TURN.into()),
386                );
387                m.metadata.insert(
388                    MNEMO_META_THREAD.into(),
389                    Value::String(thread_id.to_string()),
390                );
391                m.metadata
392                    .insert(MNEMO_META_TURN_ID.into(), Value::String(p.turn_id.clone()));
393                m.metadata
394                    .insert(MNEMO_META_SEQ.into(), Value::Number(this_seq.into()));
395                m.metadata
396                    .insert(MNEMO_META_ROLE.into(), Value::String(p.role.clone()));
397                if p.parts.len() > 1 {
398                    m.metadata.insert(
399                        MNEMO_META_PART.into(),
400                        Value::Number((part_idx as u64).into()),
401                    );
402                }
403                if p.markers_only {
404                    m.metadata
405                        .insert(MNEMO_META_MARKERS_ONLY.into(), Value::Bool(true));
406                }
407                if let Some((sha, len)) = &p.truncation {
408                    m.metadata
409                        .insert(MNEMO_META_SHA256.into(), Value::String(sha.clone()));
410                    m.metadata
411                        .insert(MNEMO_META_LENGTH.into(), Value::Number((*len).into()));
412                }
413                self.put(m)?;
414                report.added_records += 1;
415            }
416        }
417        Ok(report)
418    }
419
420    /// Export the archive layer for `thread_id` in `_mnemo.seq`
421    /// order. Each entry is the Memory as stored; chunks of a
422    /// multi-part turn come out in `_mnemo.part` order within their
423    /// `_mnemo.seq`. The record `.content` is the text that was
424    /// stored (post-chunking / post-truncation); callers that need
425    /// the pre-truncation original should round-trip via
426    /// `_mnemo.sha256` + `_mnemo.length`.
427    pub fn thread_export(&mut self, thread_id: &str) -> Result<Vec<Memory>> {
428        // R-004.2: the thread's own records, from the handle caches.
429        // FMT-9e: a cold thread's records were just decoded to warm it;
430        // reuse them instead of decoding (and decompressing) twice.
431        let (entries, mut decoded) = self.thread_entries_decoded(thread_id)?;
432        let mut out: Vec<Memory> = Vec::new();
433        for e in &entries {
434            let m = match decoded.remove(&e.id) {
435                Some(m) => self.finish_memory(e, m)?,
436                None => self.read_memory(e)?,
437            };
438            if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
439                continue;
440            }
441            if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
442                != Some(MNEMO_META_KIND_TURN)
443            {
444                continue;
445            }
446            out.push(m);
447        }
448        out.sort_by(|a, b| {
449            let seq_a = a
450                .metadata
451                .get(MNEMO_META_SEQ)
452                .and_then(|v| v.as_u64())
453                .unwrap_or(0);
454            let seq_b = b
455                .metadata
456                .get(MNEMO_META_SEQ)
457                .and_then(|v| v.as_u64())
458                .unwrap_or(0);
459            let part_a = a
460                .metadata
461                .get(MNEMO_META_PART)
462                .and_then(|v| v.as_u64())
463                .unwrap_or(0);
464            let part_b = b
465                .metadata
466                .get(MNEMO_META_PART)
467                .and_then(|v| v.as_u64())
468                .unwrap_or(0);
469            seq_a.cmp(&seq_b).then_with(|| part_a.cmp(&part_b))
470        });
471        Ok(out)
472    }
473
474    /// Export a thread's archive in the **ingest shape** (Phase 12.7d):
475    /// one [`ExportedTurn`] per source turn, in `_mnemo.seq` order, with
476    /// a chunked turn's parts joined back into one `content`. Passing
477    /// the result back to [`Mnemo::ingest_thread`] for the same thread
478    /// skips every turn; for a new thread it re-creates the same records.
479    /// This is what Python `db.thread_export`, MCP `thread_export` and
480    /// `mnemo thread export` return; [`Mnemo::thread_export`] still
481    /// returns the raw records.
482    pub fn thread_export_turns(&mut self, thread_id: &str) -> Result<Vec<ExportedTurn>> {
483        let records = self.thread_export(thread_id)?;
484        let mut out: Vec<ExportedTurn> = Vec::new();
485        let mut last_seq: Option<u64> = None;
486        for m in records {
487            let meta_str = |k: &str| {
488                m.metadata
489                    .get(k)
490                    .and_then(|v| v.as_str())
491                    .unwrap_or("")
492                    .to_string()
493            };
494            let seq = m.metadata.get(MNEMO_META_SEQ).and_then(|v| v.as_u64());
495            // Records are sorted by (seq, part), so the parts of one turn
496            // are adjacent and share a seq.
497            if seq.is_some() && seq == last_seq {
498                if let Some(prev) = out.last_mut() {
499                    prev.content.push_str(&m.content);
500                    continue;
501                }
502            }
503            last_seq = seq;
504            out.push(ExportedTurn {
505                turn_id: meta_str(MNEMO_META_TURN_ID),
506                role: meta_str(MNEMO_META_ROLE),
507                timestamp: m.created_at,
508                sha256: m
509                    .metadata
510                    .get(MNEMO_META_SHA256)
511                    .and_then(|v| v.as_str())
512                    .map(String::from),
513                original_length: m.metadata.get(MNEMO_META_LENGTH).and_then(|v| v.as_u64()),
514                content: m.content,
515            });
516        }
517        Ok(out)
518    }
519
520    /// Soft-delete every record in the archive layer of `thread_id`.
521    /// Returns the number of records marked deleted. Permanent
522    /// removal still requires a subsequent `compact_file`, same as
523    /// any `Mnemo::delete`.
524    ///
525    /// Does **not** touch `_mnemo.kind = "essence"` records or other
526    /// non-turn metadata the thread may carry — only `_mnemo.kind =
527    /// "turn"` records matching the thread id.
528    pub fn thread_forget(&mut self, thread_id: &str) -> Result<usize> {
529        if self.read_only {
530            return Err(MnemoError::ReadOnly);
531        }
532        // Collect first, then delete — can't hold catalog borrows
533        // across `self.delete`.
534        let entries = self.thread_entries(thread_id)?;
535        let mut victims: Vec<Ulid> = Vec::new();
536        for e in &entries {
537            let m = self.read_memory(e)?;
538            if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
539                continue;
540            }
541            if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
542                != Some(MNEMO_META_KIND_TURN)
543            {
544                continue;
545            }
546            victims.push(m.id);
547        }
548        let n = victims.len();
549        for id in &victims {
550            self.delete(id)?;
551        }
552        Ok(n)
553    }
554
555    // --- Phase 12.7b: essence (timeline + sections) -----------------
556}