Skip to main content

mnemo_engine/store/
continuation.rs

1//! Phase 12.7c: thread continuation (thread_start and its prelude) and commit_deltas.
2
3use super::*;
4
5impl Mnemo {
6    /// Start a new thread that continues from a previous one
7    /// (Phase 12.7c). Writes a thread-link marker record with
8    /// `_mnemo.kind = "thread_link"` + `_mnemo.thread = <new_id>` +
9    /// `_mnemo.continues = <from_id>`. The marker holds no user
10    /// content; its purpose is to let `essence(Thread(new_id))`
11    /// discover the continuation chain.
12    ///
13    /// The render for the new thread pulls the previous thread's
14    /// open section items, its `next`, and its last
15    /// [`CONTINUATION_TIMELINE_ENTRIES`] timeline entries ahead of
16    /// the new thread's own essence — the "new thread, same
17    /// experience" path.
18    pub fn thread_start(&mut self, new_thread_id: &str, from_thread_id: &str) -> Result<Ulid> {
19        if self.read_only {
20            return Err(MnemoError::ReadOnly);
21        }
22        if new_thread_id.is_empty() || from_thread_id.is_empty() {
23            return Err(MnemoError::Invalid(
24                "thread_start: 'new_thread_id' and 'from_thread_id' must not be empty".into(),
25            ));
26        }
27        if new_thread_id == from_thread_id {
28            return Err(MnemoError::Invalid(
29                "thread_start: 'new_thread_id' must differ from 'from_thread_id'".into(),
30            ));
31        }
32        // Store a tiny marker record so the continuation chain is
33        // queryable from a fresh handle. The sentinel vector passes
34        // `put`'s zero-vector guard; default recall skips link records.
35        let v = self.sentinel_vector();
36        let content = format!("thread link: {new_thread_id} continues from {from_thread_id}");
37        let mut m = Memory::new(content, MemoryType::Semantic, v).with_importance(0.1);
38        m.metadata.insert(
39            MNEMO_META_KIND.into(),
40            Value::String(MNEMO_META_KIND_THREAD_LINK.into()),
41        );
42        m.metadata.insert(
43            MNEMO_META_THREAD.into(),
44            Value::String(new_thread_id.to_string()),
45        );
46        m.metadata.insert(
47            MNEMO_META_CONTINUES.into(),
48            Value::String(from_thread_id.to_string()),
49        );
50        let id = self.put(m)?;
51        Ok(id)
52    }
53
54    /// Phase 12.7c helper: render the predecessor thread's open
55    /// items + `next` + last-N timeline entries as a "continued from"
56    /// prelude. Used by [`Mnemo::essence`] to carry forward the
57    /// prior thread's live state into a fresh thread.
58    pub(super) fn render_continuation_prelude(
59        &mut self,
60        prev_thread: &str,
61        max_bytes: usize,
62    ) -> Result<String> {
63        if max_bytes == 0 {
64            return Ok(String::new());
65        }
66        let entries = self.essence_entries()?;
67        let mut open_items: Vec<Memory> = Vec::new();
68        let mut next_items: Vec<Memory> = Vec::new();
69        let mut timeline_items: Vec<Memory> = Vec::new();
70        for e in &entries {
71            let m = self.read_memory(e)?;
72            if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
73                != Some(MNEMO_META_KIND_ESSENCE)
74            {
75                continue;
76            }
77            if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(prev_thread) {
78                continue;
79            }
80            let section = match m
81                .metadata
82                .get(MNEMO_META_SECTION)
83                .and_then(|v| v.as_str())
84                .and_then(EssenceSection::parse)
85            {
86                Some(s) => s,
87                None => continue,
88            };
89            let status = m
90                .metadata
91                .get(MNEMO_META_STATUS)
92                .and_then(|v| v.as_str())
93                .unwrap_or(essence_status::OPEN);
94            match section {
95                EssenceSection::Open if status == essence_status::OPEN => {
96                    open_items.push(m);
97                }
98                EssenceSection::Next if status == essence_status::OPEN => {
99                    next_items.push(m);
100                }
101                EssenceSection::Timeline => timeline_items.push(m),
102                _ => {}
103            }
104        }
105        // Newest-first timeline, cap to last N.
106        timeline_items.sort_by_key(|m| std::cmp::Reverse(m.created_at));
107        timeline_items.truncate(CONTINUATION_TIMELINE_ENTRIES);
108        // Oldest-first for stable chronological read in the render.
109        timeline_items.reverse();
110
111        let mut out = String::new();
112        let header = format!("## continued from `{prev_thread}`\n");
113        if header.len() > max_bytes {
114            return Ok(String::new());
115        }
116        out.push_str(&header);
117        if !open_items.is_empty() {
118            let sub = "### open\n";
119            if out.len() + sub.len() <= max_bytes {
120                out.push_str(sub);
121            }
122            for m in &open_items {
123                let line = format!("- {}\n", m.content);
124                if out.len() + line.len() > max_bytes {
125                    break;
126                }
127                out.push_str(&line);
128            }
129        }
130        if !next_items.is_empty() {
131            let sub = "### next\n";
132            if out.len() + sub.len() <= max_bytes {
133                out.push_str(sub);
134            }
135            for m in &next_items {
136                let line = format!("- {}\n", m.content);
137                if out.len() + line.len() > max_bytes {
138                    break;
139                }
140                out.push_str(&line);
141            }
142        }
143        if !timeline_items.is_empty() {
144            let sub = "### recent milestones\n";
145            if out.len() + sub.len() <= max_bytes {
146                out.push_str(sub);
147            }
148            for m in &timeline_items {
149                let line = format!("- {}\n", m.content);
150                if out.len() + line.len() > max_bytes {
151                    break;
152                }
153                out.push_str(&line);
154            }
155        }
156        if out.len() < max_bytes {
157            out.push('\n');
158        }
159        Ok(out)
160    }
161
162    /// Phase 12.7c helper: find the predecessor thread id for
163    /// `thread_id` by reading its `_mnemo.kind = "thread_link"`
164    /// marker. Returns `None` if the thread has no predecessor
165    /// (fresh thread, no `thread_start` call).
166    pub(super) fn predecessor_thread(&mut self, thread_id: &str) -> Result<Option<String>> {
167        self.ensure_key_caches()?;
168        Ok(self
169            .layers
170            .as_ref()
171            .and_then(|l| l.links.get(thread_id))
172            .map(|(from, _)| from.clone()))
173    }
174
175    /// Atomic batch write combining 12.1 facts, 12.7b essence ops,
176    /// and 12.7a archive turns in one flush (Phase 12.7c; batch
177    /// embedding added in 12.7d).
178    ///
179    /// The batch runs in four steps:
180    ///
181    /// 1. **Structural checks** on every item: required fields, size
182    ///    caps, empty keys, `thread_id` for turns, and that every
183    ///    `Update` / `Resolve` / `Remove` targets a live essence record
184    ///    the op can apply to.
185    /// 2. **One `Embedder::embed` call** for every text that needs a
186    ///    vector: facts whose `vector` is empty, essence `Add` /
187    ///    `AppendTimeline` texts (when an embedder is registered;
188    ///    without one, essence keeps its sentinel vector), and the
189    ///    archive turns' chunks.
190    /// 3. **Vector checks** (width, non-zero, finite) on supplied and
191    ///    embedded vectors alike.
192    /// 4. **Writes**: facts, then essence ops, then turns per
193    ///    `turn_policy`, then exactly one `flush()`.
194    ///
195    /// A failure in steps 1–3 returns an error with nothing staged. A
196    /// text-only fact, or any turn to archive, with no embedder
197    /// registered is a step-1 error. `put` has no rollback, so an I/O
198    /// error during step 4 can leave the batch's earlier writes staged
199    /// on the handle; steps 1–3 exist so nothing else can fail that
200    /// late.
201    pub fn commit_deltas(&mut self, batch: CommitBatch) -> Result<CommitReport> {
202        if self.read_only {
203            return Err(MnemoError::ReadOnly);
204        }
205        let CommitBatch {
206            mut facts,
207            essence_ops,
208            turns,
209            thread_id,
210            turn_policy,
211        } = batch;
212        let has_embedder = self.embedder.is_some();
213        let in_ctx = |ctx: &str, e: MnemoError| MnemoError::Invalid(format!("{ctx}: {e}"));
214
215        // --- 1. Structural checks (reads only) ----------------------
216        for (i, f) in facts.iter().enumerate() {
217            if let Some(k) = &f.key {
218                if k.is_empty() {
219                    return Err(MnemoError::Invalid(format!(
220                        "commit_deltas: facts[{i}] key is empty"
221                    )));
222                }
223            }
224            if f.memory.vector.is_empty() {
225                if f.memory.content.is_empty() {
226                    return Err(MnemoError::Invalid(format!(
227                        "commit_deltas: facts[{i}] has neither a vector nor content to embed"
228                    )));
229                }
230                if !has_embedder {
231                    return Err(MnemoError::Invalid(format!(
232                        "commit_deltas: facts[{i}] has no vector and no embedder is registered \
233                         on this handle; pass a vector or call set_embedder first"
234                    )));
235                }
236            }
237        }
238        let mut removed_in_batch: std::collections::HashSet<u128> =
239            std::collections::HashSet::new();
240        for (i, op) in essence_ops.iter().enumerate() {
241            let ctx = format!("commit_deltas: essence_ops[{i}]");
242            match op.kind {
243                EssenceOpKind::Add => {
244                    if op.scope.is_none() {
245                        return Err(MnemoError::Invalid(format!("{ctx} Add missing 'scope'")));
246                    }
247                    let section = op.section.ok_or_else(|| {
248                        MnemoError::Invalid(format!("{ctx} Add missing 'section'"))
249                    })?;
250                    let text = op
251                        .text
252                        .as_deref()
253                        .ok_or_else(|| MnemoError::Invalid(format!("{ctx} Add missing 'text'")))?;
254                    Self::check_essence_add(section, text).map_err(|e| in_ctx(&ctx, e))?;
255                }
256                EssenceOpKind::AppendTimeline => {
257                    if op.scope.is_none() {
258                        return Err(MnemoError::Invalid(format!(
259                            "{ctx} AppendTimeline missing 'scope'"
260                        )));
261                    }
262                    let text = op.text.as_deref().ok_or_else(|| {
263                        MnemoError::Invalid(format!("{ctx} AppendTimeline missing 'text'"))
264                    })?;
265                    Self::check_essence_append(text).map_err(|e| in_ctx(&ctx, e))?;
266                }
267                EssenceOpKind::Update | EssenceOpKind::Resolve | EssenceOpKind::Remove => {
268                    let id = op
269                        .id
270                        .ok_or_else(|| MnemoError::Invalid(format!("{ctx} needs 'id'")))?;
271                    if removed_in_batch.contains(&id.0) {
272                        return Err(MnemoError::Invalid(format!(
273                            "{ctx}: record {id} is removed earlier in this batch"
274                        )));
275                    }
276                    let existing = self.load_essence_record(&id).map_err(|e| in_ctx(&ctx, e))?;
277                    let section = self.section_of(&existing).map_err(|e| in_ctx(&ctx, e))?;
278                    match op.kind {
279                        EssenceOpKind::Update => {
280                            let text = op.text.as_deref().ok_or_else(|| {
281                                MnemoError::Invalid(format!("{ctx} Update needs 'text'"))
282                            })?;
283                            if section == EssenceSection::Timeline {
284                                return Err(MnemoError::Invalid(format!(
285                                    "{ctx}: timeline entries are append-only"
286                                )));
287                            }
288                            if text.is_empty() || text.len() > section.item_max_bytes() {
289                                return Err(MnemoError::Invalid(format!(
290                                    "{ctx}: '{}' items must be 1..={} bytes; got {}",
291                                    section.as_str(),
292                                    section.item_max_bytes(),
293                                    text.len()
294                                )));
295                            }
296                        }
297                        EssenceOpKind::Resolve => {
298                            if !section.has_status_lifecycle() {
299                                return Err(MnemoError::Invalid(format!(
300                                    "{ctx}: '{}' section has no status lifecycle",
301                                    section.as_str()
302                                )));
303                            }
304                        }
305                        _ => {
306                            removed_in_batch.insert(id.0);
307                        }
308                    }
309                }
310            }
311        }
312        if !turns.is_empty() && thread_id.is_none() {
313            return Err(MnemoError::Invalid(
314                "commit_deltas: 'turns' non-empty but 'thread_id' missing".into(),
315            ));
316        }
317        let archive_turns = !turns.is_empty() && !matches!(turn_policy, TurnPolicy::Drop);
318        if archive_turns && !has_embedder {
319            return Err(MnemoError::Invalid(
320                "commit_deltas: archiving 'turns' needs an embedder registered on this handle; \
321                 call set_embedder first or use turn_policy Drop"
322                    .into(),
323            ));
324        }
325        let plan = match (&thread_id, archive_turns) {
326            (Some(t), true) => Some(self.plan_ingest(t, &turns)?),
327            _ => None,
328        };
329
330        // --- 2. One batched embed -----------------------------------
331        let embeds_essence = |op: &EssenceOp| {
332            has_embedder && matches!(op.kind, EssenceOpKind::Add | EssenceOpKind::AppendTimeline)
333        };
334        let (vectors, n_texts) = {
335            let mut texts: Vec<&str> = Vec::new();
336            for f in &facts {
337                if f.memory.vector.is_empty() {
338                    texts.push(f.memory.content.as_str());
339                }
340            }
341            for op in essence_ops.iter().filter(|op| embeds_essence(op)) {
342                texts.push(op.text.as_deref().expect("checked in step 1"));
343            }
344            if let Some(p) = &plan {
345                texts.extend(p.texts());
346            }
347            let vectors = match (texts.is_empty(), self.embedder.as_ref()) {
348                (true, _) => Vec::new(),
349                (false, Some(e)) => e.embed(&texts)?,
350                (false, None) => unreachable!("texts need an embedder; checked in step 1"),
351            };
352            (vectors, texts.len())
353        };
354
355        // --- 3. Vector checks ---------------------------------------
356        self.check_vectors("commit_deltas", &vectors, n_texts)?;
357        for (i, f) in facts.iter().enumerate() {
358            let v = &f.memory.vector;
359            if v.is_empty() {
360                continue;
361            }
362            if v.len() != self.dimensions {
363                return Err(MnemoError::Invalid(format!(
364                    "commit_deltas: facts[{i}] vector has {} dims; this file is {}",
365                    v.len(),
366                    self.dimensions
367                )));
368            }
369            let ctx = format!("commit_deltas: facts[{i}]");
370            self.check_vector(&ctx, v)?;
371        }
372
373        // Hand the embedded vectors back out, in the order they were
374        // requested: facts, then essence ops, then turn chunks.
375        let mut vectors = vectors.into_iter();
376        for f in facts.iter_mut() {
377            if f.memory.vector.is_empty() {
378                f.memory.vector = vectors.next().expect("one vector per text");
379            }
380        }
381        let essence_vectors: Vec<Option<Vec<f32>>> = essence_ops
382            .iter()
383            .map(|op| embeds_essence(op).then(|| vectors.next().expect("one vector per text")))
384            .collect();
385        let turn_vectors: Vec<Vec<f32>> = vectors.collect();
386
387        // --- 4. Writes ----------------------------------------------
388        let mut report = CommitReport::default();
389
390        for f in facts {
391            match f.key {
392                Some(k) => {
393                    let (_id, outcome) = self.remember_keyed(f.memory, &k)?;
394                    match outcome {
395                        WriteOutcome::Added => report.facts_added += 1,
396                        WriteOutcome::Superseded | WriteOutcome::Updated => {
397                            report.facts_superseded += 1
398                        }
399                        WriteOutcome::Unchanged => report.facts_unchanged += 1,
400                    }
401                }
402                None => {
403                    self.remember(f.memory)?;
404                    report.facts_added += 1;
405                }
406            }
407        }
408
409        for (op, vector) in essence_ops.into_iter().zip(essence_vectors) {
410            let outcome = match op.kind {
411                EssenceOpKind::Add => {
412                    self.essence_add_with(
413                        op.scope.expect("checked in step 1"),
414                        op.section.expect("checked in step 1"),
415                        op.text.as_deref().expect("checked in step 1"),
416                        op.refs,
417                        vector,
418                    )?;
419                    WriteOutcome::Added
420                }
421                EssenceOpKind::AppendTimeline => {
422                    self.essence_append_with(
423                        op.scope.expect("checked in step 1"),
424                        op.text.as_deref().expect("checked in step 1"),
425                        op.refs,
426                        vector,
427                    )?;
428                    WriteOutcome::Added
429                }
430                EssenceOpKind::Update => self.essence_update(
431                    op.id.as_ref().expect("checked in step 1"),
432                    op.text.as_deref().expect("checked in step 1"),
433                    op.refs,
434                )?,
435                EssenceOpKind::Resolve => {
436                    self.essence_resolve(op.id.as_ref().expect("checked in step 1"))?
437                }
438                EssenceOpKind::Remove => {
439                    self.essence_remove(op.id.as_ref().expect("checked in step 1"))?;
440                    WriteOutcome::Updated
441                }
442            };
443            match outcome {
444                WriteOutcome::Unchanged => report.essence_ops_unchanged += 1,
445                _ => report.essence_ops_applied += 1,
446            }
447        }
448
449        if let (Some(plan), Some(t)) = (plan, thread_id.as_deref()) {
450            let ttl = match turn_policy {
451                TurnPolicy::ExpireAfter(secs) => Some(secs),
452                _ => None,
453            };
454            let r = self.apply_ingest(t, plan, turn_vectors, ttl)?;
455            report.turns_added += r.added_records;
456            report.turns_skipped += r.skipped_existing_turns;
457            report.turns_truncated += r.truncated_turns;
458            report.turns_chunked += r.chunked_turns;
459        }
460
461        self.flush()?;
462        Ok(report)
463    }
464}