Skip to main content

mnemo_engine/store/
essence.rs

1//! Phase 12.7b: essence items (add, append, update, resolve, remove) and the essence render.
2
3use super::*;
4
5/// Split `s` into chunks, each with `<= max_bytes` of UTF-8 content,
6/// landing on `char` boundaries. Preserves original ordering; a trailing
7/// chunk may be shorter than `max_bytes`.
8/// Phase 12.7b helper: pull the `_mnemo.status` string from an
9/// essence record. Returns `None` for timeline entries (they have no
10/// status) and for malformed records.
11pub(super) fn status_of(m: &Memory) -> Option<&str> {
12    m.metadata.get(MNEMO_META_STATUS).and_then(|v| v.as_str())
13}
14
15/// Phase 12.7b helper: render one essence item as a markdown bullet.
16/// Timeline entries get a date tag; section items get `[open]` /
17/// `[resolved]` so the reader can scan active work quickly.
18fn render_essence_item(section: EssenceSection, m: &Memory) -> String {
19    if section == EssenceSection::Timeline {
20        let date = format_unix_date(m.created_at);
21        format!("- [{date}] {}\n", m.content)
22    } else {
23        let status = status_of(m).unwrap_or(essence_status::OPEN);
24        format!("- [{status}] {}\n", m.content)
25    }
26}
27
28/// Minimal unix-seconds → `YYYY-MM-DD` formatter (no chrono/time
29/// dependency; mirrors the ISO 8601 parser in `transcript.rs`).
30fn format_unix_date(secs: i64) -> String {
31    let days_total = secs.div_euclid(86_400);
32    let (y, m, d) = civil_from_days(days_total);
33    format!("{y:04}-{m:02}-{d:02}")
34}
35
36/// Howard Hinnant's `civil_from_days` — inverse of `days_from_civil`,
37/// adequate for Gregorian dates since 1970. Pure arithmetic, no
38/// dependencies.
39fn civil_from_days(z: i64) -> (i64, u32, u32) {
40    let z = z + 719_468;
41    let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
42    let doe = (z - era * 146_097) as u64;
43    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
44    let y = yoe as i64 + era * 400;
45    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
46    let mp = (5 * doy + 2) / 153;
47    let d = (doy - (153 * mp + 2) / 5 + 1) as u32;
48    let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32;
49    let y = if m <= 2 { y + 1 } else { y };
50    (y, m, d)
51}
52
53impl Mnemo {
54    /// Phase 12.7b: add a new essence item to a section (any section
55    /// except [`EssenceSection::Timeline`]). Content ≤
56    /// [`memory::ESSENCE_SECTION_MAX_BYTES`] (280 bytes); items over
57    /// that are rejected. Starting status is `"open"`.
58    ///
59    /// Use [`Mnemo::essence_append_timeline`] for timeline entries —
60    /// they have a different cap (600 bytes), are append-only, and
61    /// carry no status lifecycle.
62    pub fn essence_add(
63        &mut self,
64        scope: EssenceScope,
65        section: EssenceSection,
66        text: &str,
67        refs: Option<Vec<String>>,
68    ) -> Result<Ulid> {
69        if self.read_only {
70            return Err(MnemoError::ReadOnly);
71        }
72        Self::check_essence_add(section, text)?;
73        self.essence_add_with(scope, section, text, refs, None)
74    }
75
76    /// [`Mnemo::essence_add`] with the item's vector supplied by the
77    /// caller instead of the registered embedder, so a caller can embed
78    /// `text` without holding the handle (the UniFFI binding does, as with
79    /// [`Mnemo::ingest_thread_with_vectors`]). Same validation, same record.
80    pub fn essence_add_with_vector(
81        &mut self,
82        scope: EssenceScope,
83        section: EssenceSection,
84        text: &str,
85        refs: Option<Vec<String>>,
86        vector: Vec<f32>,
87    ) -> Result<Ulid> {
88        if self.read_only {
89            return Err(MnemoError::ReadOnly);
90        }
91        Self::check_essence_add(section, text)?;
92        self.essence_add_with(scope, section, text, refs, Some(vector))
93    }
94
95    /// Validation shared by [`Mnemo::essence_add`] and `commit_deltas`.
96    pub(super) fn check_essence_add(section: EssenceSection, text: &str) -> Result<()> {
97        if section == EssenceSection::Timeline {
98            return Err(MnemoError::Invalid(
99                "essence_add: use essence_append_timeline for the timeline section".into(),
100            ));
101        }
102        if text.is_empty() {
103            return Err(MnemoError::Invalid(
104                "essence_add: text must not be empty".into(),
105            ));
106        }
107        if text.len() > section.item_max_bytes() {
108            return Err(MnemoError::Invalid(format!(
109                "essence_add: '{}' items must be <= {} bytes; got {}",
110                section.as_str(),
111                section.item_max_bytes(),
112                text.len()
113            )));
114        }
115        Ok(())
116    }
117
118    /// Write half of `essence_add`, with an optional precomputed vector.
119    pub(super) fn essence_add_with(
120        &mut self,
121        scope: EssenceScope,
122        section: EssenceSection,
123        text: &str,
124        refs: Option<Vec<String>>,
125        vector: Option<Vec<f32>>,
126    ) -> Result<Ulid> {
127        let now = memory::now_secs();
128        let mem = self.build_essence_record(scope, section, text, refs, Some(now), vector)?;
129        self.put(mem)
130    }
131
132    /// Phase 12.7b: append a new timeline entry (append-only
133    /// milestone). Content ≤ [`memory::ESSENCE_TIMELINE_MAX_BYTES`]
134    /// (600 bytes); entries over that are rejected (not truncated —
135    /// truncating a milestone silently loses the narrative).
136    pub fn essence_append_timeline(
137        &mut self,
138        scope: EssenceScope,
139        text: &str,
140        refs: Option<Vec<String>>,
141    ) -> Result<Ulid> {
142        if self.read_only {
143            return Err(MnemoError::ReadOnly);
144        }
145        Self::check_essence_append(text)?;
146        self.essence_append_with(scope, text, refs, None)
147    }
148
149    /// [`Mnemo::essence_append_timeline`] with a caller-supplied vector
150    /// (see [`Mnemo::essence_add_with_vector`]).
151    pub fn essence_append_timeline_with_vector(
152        &mut self,
153        scope: EssenceScope,
154        text: &str,
155        refs: Option<Vec<String>>,
156        vector: Vec<f32>,
157    ) -> Result<Ulid> {
158        if self.read_only {
159            return Err(MnemoError::ReadOnly);
160        }
161        Self::check_essence_append(text)?;
162        self.essence_append_with(scope, text, refs, Some(vector))
163    }
164
165    /// Validation shared by [`Mnemo::essence_append_timeline`] and
166    /// `commit_deltas`.
167    pub(super) fn check_essence_append(text: &str) -> Result<()> {
168        if text.is_empty() {
169            return Err(MnemoError::Invalid(
170                "essence_append_timeline: text must not be empty".into(),
171            ));
172        }
173        if text.len() > EssenceSection::Timeline.item_max_bytes() {
174            return Err(MnemoError::Invalid(format!(
175                "essence_append_timeline: entries must be <= {} bytes; got {}",
176                EssenceSection::Timeline.item_max_bytes(),
177                text.len()
178            )));
179        }
180        Ok(())
181    }
182
183    /// Write half of `essence_append_timeline`, with an optional
184    /// precomputed vector.
185    pub(super) fn essence_append_with(
186        &mut self,
187        scope: EssenceScope,
188        text: &str,
189        refs: Option<Vec<String>>,
190        vector: Option<Vec<f32>>,
191    ) -> Result<Ulid> {
192        let mem =
193            self.build_essence_record(scope, EssenceSection::Timeline, text, refs, None, vector)?;
194        self.put(mem)
195    }
196
197    /// Phase 12.7b: update an existing essence section item in place.
198    /// Returns [`WriteOutcome::Updated`] / [`WriteOutcome::Unchanged`].
199    /// Timeline entries are append-only — calling this on a timeline
200    /// record is an error.
201    pub fn essence_update(
202        &mut self,
203        id: &Ulid,
204        text: &str,
205        refs: Option<Vec<String>>,
206    ) -> Result<WriteOutcome> {
207        if self.read_only {
208            return Err(MnemoError::ReadOnly);
209        }
210        let existing = self.load_essence_record(id)?;
211        let section = self.section_of(&existing)?;
212        if section == EssenceSection::Timeline {
213            return Err(MnemoError::Invalid(
214                "essence_update: timeline entries are append-only; append a new one instead".into(),
215            ));
216        }
217        if text.is_empty() {
218            return Err(MnemoError::Invalid(
219                "essence_update: text must not be empty".into(),
220            ));
221        }
222        if text.len() > section.item_max_bytes() {
223            return Err(MnemoError::Invalid(format!(
224                "essence_update: '{}' items must be <= {} bytes; got {}",
225                section.as_str(),
226                section.item_max_bytes(),
227                text.len()
228            )));
229        }
230        let mut patch = MemoryPatch {
231            content: Some(text.to_string()),
232            ..Default::default()
233        };
234        // We don't have a usable embedder-free path for an in-place
235        // content update, so provide the same vector as the existing
236        // record (the text-vector drift guard in `update` will accept
237        // this).
238        patch.vector = Some(existing.vector.clone());
239        if let Some(r) = refs {
240            let mut meta = Map::new();
241            // Preserve the user-visible metadata (none for essence —
242            // everything lives under `_mnemo.*`) and let `update`'s
243            // metadata-replacement path keep the reserved keys.
244            meta.insert(
245                MNEMO_META_REFS.into(),
246                Value::Array(r.into_iter().map(Value::String).collect()),
247            );
248            patch.metadata = Some(meta);
249        }
250        self.update(id, patch)
251    }
252
253    /// Phase 12.7b: mark a section item as resolved (open → resolved
254    /// transition). A new history entry lands on
255    /// `_mnemo.status_history`. Idempotent — resolving a record that
256    /// is already resolved is a no-op.
257    pub fn essence_resolve(&mut self, id: &Ulid) -> Result<WriteOutcome> {
258        self.essence_transition_status(id, essence_status::RESOLVED)
259    }
260
261    /// Phase 12.7b: soft-delete a section item. Same semantics as
262    /// [`Mnemo::delete`] on any record — space reclaimed by a later
263    /// compact.
264    pub fn essence_remove(&mut self, id: &Ulid) -> Result<()> {
265        if self.read_only {
266            return Err(MnemoError::ReadOnly);
267        }
268        self.delete(id)
269    }
270
271    /// Phase 12.7b: deterministic, budgeted markdown render of the
272    /// essence layer for a thread (or the project layer when
273    /// `scope = EssenceScope::Project`). `max_bytes` caps the output;
274    /// sections are emitted in the fixed [`EssenceSection::render_order`],
275    /// and each section's own budget is proportional to its position
276    /// in the order. Oldest timeline entries fold into a
277    /// `+N earlier milestones` tail when the budget runs out.
278    ///
279    /// The render always ends with an archive pointer so the reader
280    /// knows how to drill in deeper (`mnemo thread export …` or
281    /// `recall --thread …`).
282    pub fn essence(&mut self, scope: EssenceScope, max_bytes: usize) -> Result<String> {
283        let max_bytes = if max_bytes == 0 {
284            ESSENCE_DEFAULT_RENDER_BYTES
285        } else {
286            max_bytes
287        };
288        // Collect every live essence record for this scope.
289        // R-004.2: only the essence items, from the handle caches.
290        let entries = self.essence_entries()?;
291        let mut by_section: std::collections::BTreeMap<&'static str, Vec<Memory>> =
292            std::collections::BTreeMap::new();
293        for section in EssenceSection::render_order() {
294            by_section.insert(section.as_str(), Vec::new());
295        }
296        for e in &entries {
297            let m = self.read_memory(e)?;
298            if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
299                != Some(MNEMO_META_KIND_ESSENCE)
300            {
301                continue;
302            }
303            // Scope filter. Project-scope records are shown in every
304            // thread's render (and in the project render); thread-scope
305            // records only match when scope == same thread.
306            let rec_thread = m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str());
307            let matches_scope = match (&scope, rec_thread) {
308                (EssenceScope::Project, None) => true,
309                (EssenceScope::Thread(_), None) => true, // project-scope always shows
310                (EssenceScope::Thread(t), Some(rt)) => rt == t,
311                (EssenceScope::Project, Some(_)) => false,
312            };
313            if !matches_scope {
314                continue;
315            }
316            let section_str = m
317                .metadata
318                .get(MNEMO_META_SECTION)
319                .and_then(|v| v.as_str())
320                .unwrap_or("timeline");
321            by_section
322                .entry(
323                    EssenceSection::parse(section_str)
324                        .map(|s| s.as_str())
325                        .unwrap_or("timeline"),
326                )
327                .or_default()
328                .push(m);
329        }
330
331        // Sort each section's records:
332        // - Decisions + Timeline: newest-first (by created_at desc).
333        // - Everything else: sort open items before resolved, then
334        //   newest-first. Cheap, deterministic.
335        for (name, items) in by_section.iter_mut() {
336            let newest_first = matches!(*name, "decisions" | "timeline");
337            items.sort_by(|a, b| {
338                let a_resolved = status_of(a) == Some(essence_status::RESOLVED);
339                let b_resolved = status_of(b) == Some(essence_status::RESOLVED);
340                match (a_resolved, b_resolved) {
341                    (false, true) => std::cmp::Ordering::Less,
342                    (true, false) => std::cmp::Ordering::Greater,
343                    _ => {
344                        if newest_first {
345                            b.created_at.cmp(&a.created_at)
346                        } else {
347                            a.created_at.cmp(&b.created_at)
348                        }
349                    }
350                }
351            });
352        }
353
354        // Per-section budget: give intent / conventions / decisions a
355        // larger share (they frame the thread), timeline a smaller
356        // share (it can fold). This is deterministic because it
357        // depends only on `max_bytes`.
358        let header_reserve = 256usize; // header + archive pointer
359        let body_budget = max_bytes.saturating_sub(header_reserve);
360        let per_section_budget = |section: EssenceSection| -> usize {
361            match section {
362                EssenceSection::Intent => body_budget / 8,
363                EssenceSection::Conventions => body_budget / 6,
364                EssenceSection::Decisions => body_budget / 4,
365                EssenceSection::State => body_budget / 8,
366                EssenceSection::Open => body_budget / 8,
367                EssenceSection::Next => body_budget / 12,
368                EssenceSection::Lessons => body_budget / 12,
369                EssenceSection::Timeline => body_budget / 4,
370            }
371        };
372
373        let mut out = String::with_capacity(max_bytes);
374        let scope_label = match &scope {
375            EssenceScope::Thread(t) => format!("thread `{t}`"),
376            EssenceScope::Project => "project".to_string(),
377        };
378        out.push_str(&format!("# essence — {scope_label}\n\n"));
379
380        // Phase 12.7c: continuation. If this is a thread render AND
381        // the thread has a `_mnemo.continues` link, pull the prior
382        // thread's open section items, its `next`, and its last-N
383        // timeline entries in ahead of the new thread's own essence.
384        if let EssenceScope::Thread(new_id) = &scope {
385            if let Some(prev_id) = self.predecessor_thread(new_id)? {
386                let continuation = self
387                    .render_continuation_prelude(&prev_id, max_bytes.saturating_sub(out.len()))?;
388                if !continuation.is_empty() && out.len() + continuation.len() <= max_bytes {
389                    out.push_str(&continuation);
390                }
391            }
392        }
393
394        for section in EssenceSection::render_order() {
395            let items = match by_section.get(section.as_str()) {
396                Some(v) if !v.is_empty() => v,
397                _ => continue,
398            };
399            let budget = per_section_budget(*section);
400            let section_header = format!("## {}\n", section.as_str());
401            if out.len() + section_header.len() > max_bytes {
402                break;
403            }
404            out.push_str(&section_header);
405            let mut used = 0usize;
406            let mut rendered = 0usize;
407            for item in items {
408                let line = render_essence_item(*section, item);
409                if used + line.len() > budget {
410                    break;
411                }
412                if out.len() + line.len() > max_bytes {
413                    break;
414                }
415                out.push_str(&line);
416                used += line.len();
417                rendered += 1;
418            }
419            // Fold the remainder for timeline.
420            if *section == EssenceSection::Timeline && rendered < items.len() {
421                let fold = format!("- +{} earlier milestones\n", items.len() - rendered);
422                if out.len() + fold.len() <= max_bytes {
423                    out.push_str(&fold);
424                }
425            }
426            if out.len() < max_bytes {
427                out.push('\n');
428            }
429        }
430
431        // Archive pointer.
432        let pointer = match &scope {
433            EssenceScope::Thread(t) => format!(
434                "> full history: `mnemo thread export <file> --thread {t}` or `recall --thread {t}`\n"
435            ),
436            EssenceScope::Project => String::from(
437                "> full history: `mnemo thread export <file> --thread <id>` or `recall --thread <id>`\n",
438            ),
439        };
440        if out.len() + pointer.len() <= max_bytes {
441            out.push_str(&pointer);
442        }
443        Ok(out)
444    }
445
446    // --- essence internal helpers ----------------------------------
447
448    /// Build an essence record. `vector` is a precomputed embedding (the
449    /// 12.7d `commit_deltas` batch path); `None` embeds `text` with the
450    /// registered embedder, or falls back to an axis-neutral unit vector
451    /// when there is none. Essence records are read by `_mnemo.kind` and
452    /// scope, not by similarity, so the fallback costs no retrieval
453    /// quality; it only has to pass `put`'s non-zero guard.
454    pub(super) fn build_essence_record(
455        &self,
456        scope: EssenceScope,
457        section: EssenceSection,
458        text: &str,
459        refs: Option<Vec<String>>,
460        status_at: Option<i64>,
461        vector: Option<Vec<f32>>,
462    ) -> Result<Memory> {
463        let v = match (vector, self.embedder.as_ref()) {
464            (Some(v), _) => v,
465            (None, Some(e)) => e.embed(&[text])?.pop().ok_or_else(|| {
466                MnemoError::Invalid("essence: embedder returned no vector".into())
467            })?,
468            (None, None) => self.sentinel_vector(),
469        };
470        let mut m = Memory::new(text.to_string(), MemoryType::Semantic, v).with_importance(0.5);
471        m.metadata.insert(
472            MNEMO_META_KIND.into(),
473            Value::String(MNEMO_META_KIND_ESSENCE.into()),
474        );
475        m.metadata.insert(
476            MNEMO_META_SECTION.into(),
477            Value::String(section.as_str().into()),
478        );
479        if let EssenceScope::Thread(t) = &scope {
480            m.metadata
481                .insert(MNEMO_META_THREAD.into(), Value::String(t.clone()));
482        }
483        if let Some(r) = refs {
484            m.metadata.insert(
485                MNEMO_META_REFS.into(),
486                Value::Array(r.into_iter().map(Value::String).collect()),
487            );
488        }
489        if let Some(at) = status_at {
490            m.metadata.insert(
491                MNEMO_META_STATUS.into(),
492                Value::String(essence_status::OPEN.into()),
493            );
494            m.metadata.insert(
495                MNEMO_META_STATUS_HISTORY.into(),
496                Value::Array(vec![serde_json::json!({
497                    "status": essence_status::OPEN,
498                    "at": at,
499                })]),
500            );
501        }
502        Ok(m)
503    }
504
505    /// Axis-neutral unit vector (every component `1/sqrt(dims)`), used by
506    /// essence records without an embedder and by thread-link markers.
507    pub(super) fn sentinel_vector(&self) -> Vec<f32> {
508        let dims = self.dimensions.max(1);
509        vec![1.0 / (dims as f32).sqrt(); dims]
510    }
511
512    pub(super) fn load_essence_record(&mut self, id: &Ulid) -> Result<Memory> {
513        let m = self.get(id)?;
514        if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str()) != Some(MNEMO_META_KIND_ESSENCE)
515        {
516            return Err(MnemoError::Invalid(format!(
517                "essence: record {id} is not an essence record (missing `_mnemo.kind = essence`)"
518            )));
519        }
520        Ok(m)
521    }
522
523    pub(super) fn section_of(&self, m: &Memory) -> Result<EssenceSection> {
524        let s = m
525            .metadata
526            .get(MNEMO_META_SECTION)
527            .and_then(|v| v.as_str())
528            .ok_or_else(|| {
529                MnemoError::Invalid("essence: record missing `_mnemo.section`".into())
530            })?;
531        EssenceSection::parse(s)
532            .ok_or_else(|| MnemoError::Invalid(format!("essence: unknown section '{s}' on record")))
533    }
534
535    pub(super) fn essence_transition_status(
536        &mut self,
537        id: &Ulid,
538        to: &str,
539    ) -> Result<WriteOutcome> {
540        if self.read_only {
541            return Err(MnemoError::ReadOnly);
542        }
543        let existing = self.load_essence_record(id)?;
544        let section = self.section_of(&existing)?;
545        if !section.has_status_lifecycle() {
546            return Err(MnemoError::Invalid(format!(
547                "essence_transition: '{}' section has no status lifecycle",
548                section.as_str()
549            )));
550        }
551        let from = existing
552            .metadata
553            .get(MNEMO_META_STATUS)
554            .and_then(|v| v.as_str())
555            .unwrap_or(essence_status::OPEN);
556        if from == to {
557            return Ok(WriteOutcome::Unchanged);
558        }
559        let allowed = ESSENCE_TRANSITIONS
560            .iter()
561            .any(|(f, t)| *f == from && *t == to);
562        if !allowed {
563            return Err(MnemoError::Invalid(format!(
564                "essence_transition: '{from}' → '{to}' is not a valid transition for essence"
565            )));
566        }
567        // Mutate metadata in place and `put` back under the same id
568        // so `_mnemo.updated_at` gets stamped and access stats are
569        // preserved.
570        let mut new = existing.clone();
571        new.metadata
572            .insert(MNEMO_META_STATUS.into(), Value::String(to.to_string()));
573        let now = memory::now_secs();
574        let history_entry = serde_json::json!({ "status": to, "at": now });
575        let history = new
576            .metadata
577            .entry(MNEMO_META_STATUS_HISTORY.to_string())
578            .or_insert_with(|| Value::Array(Vec::new()));
579        if let Value::Array(arr) = history {
580            arr.push(history_entry);
581        }
582        new.metadata
583            .insert(MNEMO_META_UPDATED_AT.into(), Value::Number(now.into()));
584        new.id = existing.id;
585        new.created_at = existing.created_at;
586        self.put(new)?;
587        Ok(WriteOutcome::Updated)
588    }
589
590    // --- Phase 12.7c: continuation + commit_deltas ------------------
591}