Skip to main content

mnemo_engine/store/
essence.rs

1//! Phase 12.7b: essence items (add, append, update, resolve, remove) and the essence render.
2
3use super::*;
4
5/// Split `s` into chunks, each with `<= max_bytes` of UTF-8 content,
6/// landing on `char` boundaries. Preserves original ordering; a trailing
7/// chunk may be shorter than `max_bytes`.
8/// Phase 12.7b helper: pull the `_mnemo.status` string from an
9/// essence record. Returns `None` for timeline entries (they have no
10/// status) and for malformed records.
11pub(super) fn status_of(m: &Memory) -> Option<&str> {
12    m.metadata.get(MNEMO_META_STATUS).and_then(|v| v.as_str())
13}
14
15/// Phase 12.7b helper: render one essence item as a markdown bullet.
16/// Timeline entries get a date tag; section items get `[open]` /
17/// `[resolved]` so the reader can scan active work quickly.
18fn render_essence_item(section: EssenceSection, m: &Memory, now: i64) -> String {
19    if section == EssenceSection::Timeline {
20        let date = format_unix_date(m.created_at);
21        format!("- [{date}] {}\n", m.content)
22    } else {
23        let status = status_of(m).unwrap_or(essence_status::OPEN);
24        // 13a (13.6): an item past its review date, or its section's age.
25        let stale = crate::memory::stale_since(m, now)
26            .map(|t| format!(" [stale since {}]", format_unix_date(t)))
27            .unwrap_or_default();
28        format!("- [{status}] {}{stale}\n", m.content)
29    }
30}
31
32/// Minimal unix-seconds → `YYYY-MM-DD` formatter (no chrono/time
33/// dependency; mirrors the ISO 8601 parser in `transcript.rs`).
34pub(crate) fn format_unix_date(secs: i64) -> String {
35    let days_total = secs.div_euclid(86_400);
36    let (y, m, d) = civil_from_days(days_total);
37    format!("{y:04}-{m:02}-{d:02}")
38}
39
40/// Howard Hinnant's `civil_from_days` — inverse of `days_from_civil`,
41/// adequate for Gregorian dates since 1970. Pure arithmetic, no
42/// dependencies.
43fn civil_from_days(z: i64) -> (i64, u32, u32) {
44    let z = z + 719_468;
45    let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
46    let doe = (z - era * 146_097) as u64;
47    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
48    let y = yoe as i64 + era * 400;
49    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
50    let mp = (5 * doy + 2) / 153;
51    let d = (doy - (153 * mp + 2) / 5 + 1) as u32;
52    let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32;
53    let y = if m <= 2 { y + 1 } else { y };
54    (y, m, d)
55}
56
57impl Mnemo {
58    /// Phase 12.7b: add a new essence item to a section (any section
59    /// except [`EssenceSection::Timeline`]). Content ≤
60    /// [`memory::ESSENCE_SECTION_MAX_BYTES`] (280 bytes); items over
61    /// that are rejected. Starting status is `"open"`.
62    ///
63    /// Use [`Mnemo::essence_append_timeline`] for timeline entries —
64    /// they have a different cap (600 bytes), are append-only, and
65    /// carry no status lifecycle.
66    pub fn essence_add(
67        &mut self,
68        scope: EssenceScope,
69        section: EssenceSection,
70        text: &str,
71        refs: Option<Vec<String>>,
72    ) -> Result<Ulid> {
73        if self.read_only {
74            return Err(MnemoError::ReadOnly);
75        }
76        Self::check_essence_add(section, text)?;
77        self.essence_add_with(scope, section, text, refs, None)
78    }
79
80    /// [`Mnemo::essence_add`] with the item's vector supplied by the
81    /// caller instead of the registered embedder, so a caller can embed
82    /// `text` without holding the handle (the UniFFI binding does, as with
83    /// [`Mnemo::ingest_thread_with_vectors`]). Same validation, same record.
84    pub fn essence_add_with_vector(
85        &mut self,
86        scope: EssenceScope,
87        section: EssenceSection,
88        text: &str,
89        refs: Option<Vec<String>>,
90        vector: Vec<f32>,
91    ) -> Result<Ulid> {
92        if self.read_only {
93            return Err(MnemoError::ReadOnly);
94        }
95        Self::check_essence_add(section, text)?;
96        self.essence_add_with(scope, section, text, refs, Some(vector))
97    }
98
99    /// Validation shared by [`Mnemo::essence_add`] and `commit_deltas`.
100    pub(super) fn check_essence_add(section: EssenceSection, text: &str) -> Result<()> {
101        if section == EssenceSection::Timeline {
102            return Err(MnemoError::Invalid(
103                "essence_add: use essence_append_timeline for the timeline section".into(),
104            ));
105        }
106        if text.is_empty() {
107            return Err(MnemoError::Invalid(
108                "essence_add: text must not be empty".into(),
109            ));
110        }
111        if text.len() > section.item_max_bytes() {
112            return Err(MnemoError::Invalid(format!(
113                "essence_add: '{}' items must be <= {} bytes; got {}",
114                section.as_str(),
115                section.item_max_bytes(),
116                text.len()
117            )));
118        }
119        Ok(())
120    }
121
122    /// Write half of `essence_add`, with an optional precomputed vector.
123    pub(super) fn essence_add_with(
124        &mut self,
125        scope: EssenceScope,
126        section: EssenceSection,
127        text: &str,
128        refs: Option<Vec<String>>,
129        vector: Option<Vec<f32>>,
130    ) -> Result<Ulid> {
131        let now = memory::now_secs();
132        let mem = self.build_essence_record(scope, section, text, refs, Some(now), vector)?;
133        self.put(mem)
134    }
135
136    /// Phase 12.7b: append a new timeline entry (append-only
137    /// milestone). Content ≤ [`memory::ESSENCE_TIMELINE_MAX_BYTES`]
138    /// (600 bytes); entries over that are rejected (not truncated —
139    /// truncating a milestone silently loses the narrative).
140    pub fn essence_append_timeline(
141        &mut self,
142        scope: EssenceScope,
143        text: &str,
144        refs: Option<Vec<String>>,
145    ) -> Result<Ulid> {
146        if self.read_only {
147            return Err(MnemoError::ReadOnly);
148        }
149        Self::check_essence_append(text)?;
150        self.essence_append_with(scope, text, refs, None)
151    }
152
153    /// [`Mnemo::essence_append_timeline`] with a caller-supplied vector
154    /// (see [`Mnemo::essence_add_with_vector`]).
155    pub fn essence_append_timeline_with_vector(
156        &mut self,
157        scope: EssenceScope,
158        text: &str,
159        refs: Option<Vec<String>>,
160        vector: Vec<f32>,
161    ) -> Result<Ulid> {
162        if self.read_only {
163            return Err(MnemoError::ReadOnly);
164        }
165        Self::check_essence_append(text)?;
166        self.essence_append_with(scope, text, refs, Some(vector))
167    }
168
169    /// Validation shared by [`Mnemo::essence_append_timeline`] and
170    /// `commit_deltas`.
171    pub(super) fn check_essence_append(text: &str) -> Result<()> {
172        if text.is_empty() {
173            return Err(MnemoError::Invalid(
174                "essence_append_timeline: text must not be empty".into(),
175            ));
176        }
177        if text.len() > EssenceSection::Timeline.item_max_bytes() {
178            return Err(MnemoError::Invalid(format!(
179                "essence_append_timeline: entries must be <= {} bytes; got {}",
180                EssenceSection::Timeline.item_max_bytes(),
181                text.len()
182            )));
183        }
184        Ok(())
185    }
186
187    /// Write half of `essence_append_timeline`, with an optional
188    /// precomputed vector.
189    pub(super) fn essence_append_with(
190        &mut self,
191        scope: EssenceScope,
192        text: &str,
193        refs: Option<Vec<String>>,
194        vector: Option<Vec<f32>>,
195    ) -> Result<Ulid> {
196        let mem =
197            self.build_essence_record(scope, EssenceSection::Timeline, text, refs, None, vector)?;
198        self.put(mem)
199    }
200
201    /// Phase 12.7b: update an existing essence section item in place.
202    /// Returns [`WriteOutcome::Updated`] / [`WriteOutcome::Unchanged`].
203    /// Timeline entries are append-only — calling this on a timeline
204    /// record is an error.
205    pub fn essence_update(
206        &mut self,
207        id: &Ulid,
208        text: &str,
209        refs: Option<Vec<String>>,
210    ) -> Result<WriteOutcome> {
211        if self.read_only {
212            return Err(MnemoError::ReadOnly);
213        }
214        let existing = self.load_essence_record(id)?;
215        let section = self.section_of(&existing)?;
216        if section == EssenceSection::Timeline {
217            return Err(MnemoError::Invalid(
218                "essence_update: timeline entries are append-only; append a new one instead".into(),
219            ));
220        }
221        if text.is_empty() {
222            return Err(MnemoError::Invalid(
223                "essence_update: text must not be empty".into(),
224            ));
225        }
226        if text.len() > section.item_max_bytes() {
227            return Err(MnemoError::Invalid(format!(
228                "essence_update: '{}' items must be <= {} bytes; got {}",
229                section.as_str(),
230                section.item_max_bytes(),
231                text.len()
232            )));
233        }
234        let mut patch = MemoryPatch {
235            content: Some(text.to_string()),
236            ..Default::default()
237        };
238        // We don't have a usable embedder-free path for an in-place
239        // content update, so provide the same vector as the existing
240        // record (the text-vector drift guard in `update` will accept
241        // this).
242        patch.vector = Some(existing.vector.clone());
243        if let Some(r) = refs {
244            let mut meta = Map::new();
245            // Preserve the user-visible metadata (none for essence —
246            // everything lives under `_mnemo.*`) and let `update`'s
247            // metadata-replacement path keep the reserved keys.
248            meta.insert(
249                MNEMO_META_REFS.into(),
250                Value::Array(r.into_iter().map(Value::String).collect()),
251            );
252            patch.metadata = Some(meta);
253        }
254        self.update_unchecked(id, patch)
255    }
256
257    /// Phase 12.7b: mark a section item as resolved (open → resolved
258    /// transition). A new history entry lands on
259    /// `_mnemo.status_history`. Idempotent — resolving a record that
260    /// is already resolved is a no-op.
261    pub fn essence_resolve(&mut self, id: &Ulid) -> Result<WriteOutcome> {
262        self.essence_transition_status(id, essence_status::RESOLVED)
263    }
264
265    /// Phase 12.7b: soft-delete a section item, as [`Mnemo::delete`] does
266    /// (space reclaimed by a later compact). Like `essence_update` and
267    /// `essence_resolve`, it refuses a record that isn't an essence item
268    /// (RB-014).
269    pub fn essence_remove(&mut self, id: &Ulid) -> Result<()> {
270        if self.read_only {
271            return Err(MnemoError::ReadOnly);
272        }
273        self.load_essence_record(id)?;
274        self.delete(id)
275    }
276
277    /// Phase 12.7b: deterministic, budgeted markdown render of the
278    /// essence layer for a thread (or the project layer when
279    /// `scope = EssenceScope::Project`). `max_bytes` caps the output;
280    /// sections are emitted in the fixed [`EssenceSection::render_order`],
281    /// and each section's own budget is proportional to its position
282    /// in the order. Oldest timeline entries fold into a
283    /// `+N earlier milestones` tail when the budget runs out.
284    ///
285    /// The render always ends with an archive pointer so the reader
286    /// knows how to drill in deeper (`mnemo thread export …` or
287    /// `recall --thread …`).
288    pub fn essence(&mut self, scope: EssenceScope, max_bytes: usize) -> Result<String> {
289        let max_bytes = if max_bytes == 0 {
290            ESSENCE_DEFAULT_RENDER_BYTES
291        } else {
292            max_bytes
293        };
294        // Collect every live essence record for this scope.
295        // R-004.2: only the essence items, from the handle caches.
296        let entries = self.essence_entries()?;
297        let now = memory::now_secs();
298        let mut by_section: std::collections::BTreeMap<&'static str, Vec<Memory>> =
299            std::collections::BTreeMap::new();
300        for section in EssenceSection::render_order() {
301            by_section.insert(section.as_str(), Vec::new());
302        }
303        for e in &entries {
304            let m = self.read_memory(e)?;
305            if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
306                != Some(MNEMO_META_KIND_ESSENCE)
307            {
308                continue;
309            }
310            // Scope filter. Project-scope records are shown in every
311            // thread's render (and in the project render); thread-scope
312            // records only match when scope == same thread.
313            let rec_thread = m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str());
314            let matches_scope = match (&scope, rec_thread) {
315                (EssenceScope::Project, None) => true,
316                (EssenceScope::Thread(_), None) => true, // project-scope always shows
317                (EssenceScope::Thread(t), Some(rt)) => rt == t,
318                (EssenceScope::Project, Some(_)) => false,
319            };
320            if !matches_scope {
321                continue;
322            }
323            // 13a (13.5): the render shows public and team items.
324            if !crate::memory::VisibilitySet::DEFAULT.shows(&m) {
325                continue;
326            }
327            let section_str = m
328                .metadata
329                .get(MNEMO_META_SECTION)
330                .and_then(|v| v.as_str())
331                .unwrap_or("timeline");
332            by_section
333                .entry(
334                    EssenceSection::parse(section_str)
335                        .map(|s| s.as_str())
336                        .unwrap_or("timeline"),
337                )
338                .or_default()
339                .push(m);
340        }
341
342        // Sort each section's records:
343        // - Decisions + Timeline: newest-first (by created_at desc).
344        // - Everything else: sort open items before resolved, then
345        //   newest-first. Cheap, deterministic.
346        for (name, items) in by_section.iter_mut() {
347            let newest_first = matches!(*name, "decisions" | "timeline");
348            items.sort_by(|a, b| {
349                let a_resolved = status_of(a) == Some(essence_status::RESOLVED);
350                let b_resolved = status_of(b) == Some(essence_status::RESOLVED);
351                match (a_resolved, b_resolved) {
352                    (false, true) => std::cmp::Ordering::Less,
353                    (true, false) => std::cmp::Ordering::Greater,
354                    _ => {
355                        if newest_first {
356                            b.created_at.cmp(&a.created_at)
357                        } else {
358                            a.created_at.cmp(&b.created_at)
359                        }
360                    }
361                }
362            });
363        }
364
365        // Per-section budget: give intent / conventions / decisions a
366        // larger share (they frame the thread), timeline a smaller
367        // share (it can fold). This is deterministic because it
368        // depends only on `max_bytes`.
369        let header_reserve = 256usize; // header + archive pointer
370        let body_budget = max_bytes.saturating_sub(header_reserve);
371        let per_section_budget = |section: EssenceSection| -> usize {
372            match section {
373                EssenceSection::Intent => body_budget / 8,
374                EssenceSection::Conventions => body_budget / 6,
375                EssenceSection::Decisions => body_budget / 4,
376                EssenceSection::State => body_budget / 8,
377                EssenceSection::Open => body_budget / 8,
378                EssenceSection::Next => body_budget / 12,
379                EssenceSection::Lessons => body_budget / 12,
380                EssenceSection::Timeline => body_budget / 4,
381            }
382        };
383
384        let mut out = String::with_capacity(max_bytes);
385        let scope_label = match &scope {
386            EssenceScope::Thread(t) => format!("thread `{t}`"),
387            EssenceScope::Project => "project".to_string(),
388        };
389        out.push_str(&format!("# essence — {scope_label}\n\n"));
390
391        // Phase 12.7c: continuation. If this is a thread render AND
392        // the thread has a `_mnemo.continues` link, pull the prior
393        // thread's open section items, its `next`, and its last-N
394        // timeline entries in ahead of the new thread's own essence.
395        if let EssenceScope::Thread(new_id) = &scope {
396            if let Some(prev_id) = self.predecessor_thread(new_id)? {
397                let continuation = self
398                    .render_continuation_prelude(&prev_id, max_bytes.saturating_sub(out.len()))?;
399                if !continuation.is_empty() && out.len() + continuation.len() <= max_bytes {
400                    out.push_str(&continuation);
401                }
402            }
403        }
404
405        for section in EssenceSection::render_order() {
406            let items = match by_section.get(section.as_str()) {
407                Some(v) if !v.is_empty() => v,
408                _ => continue,
409            };
410            let budget = per_section_budget(*section);
411            let section_header = format!("## {}\n", section.as_str());
412            if out.len() + section_header.len() > max_bytes {
413                break;
414            }
415            out.push_str(&section_header);
416            let mut used = 0usize;
417            let mut rendered = 0usize;
418            for item in items {
419                let line = render_essence_item(*section, item, now);
420                if used + line.len() > budget {
421                    break;
422                }
423                if out.len() + line.len() > max_bytes {
424                    break;
425                }
426                out.push_str(&line);
427                used += line.len();
428                rendered += 1;
429            }
430            // Fold the remainder for timeline.
431            if *section == EssenceSection::Timeline && rendered < items.len() {
432                let fold = format!("- +{} earlier milestones\n", items.len() - rendered);
433                if out.len() + fold.len() <= max_bytes {
434                    out.push_str(&fold);
435                }
436            }
437            if out.len() < max_bytes {
438                out.push('\n');
439            }
440        }
441
442        // Archive pointer.
443        let pointer = match &scope {
444            EssenceScope::Thread(t) => format!(
445                "> full history: `mnemo thread export <file> --thread {t}` or `recall --thread {t}`\n"
446            ),
447            EssenceScope::Project => String::from(
448                "> full history: `mnemo thread export <file> --thread <id>` or `recall --thread <id>`\n",
449            ),
450        };
451        if out.len() + pointer.len() <= max_bytes {
452            out.push_str(&pointer);
453        }
454        Ok(out)
455    }
456
457    // --- essence internal helpers ----------------------------------
458
459    /// Build an essence record. `vector` is a precomputed embedding (the
460    /// 12.7d `commit_deltas` batch path); `None` embeds `text` with the
461    /// registered embedder, or falls back to an axis-neutral unit vector
462    /// when there is none. Essence records are read by `_mnemo.kind` and
463    /// scope, not by similarity, so the fallback costs no retrieval
464    /// quality; it only has to pass `put`'s non-zero guard.
465    pub(super) fn build_essence_record(
466        &self,
467        scope: EssenceScope,
468        section: EssenceSection,
469        text: &str,
470        refs: Option<Vec<String>>,
471        status_at: Option<i64>,
472        vector: Option<Vec<f32>>,
473    ) -> Result<Memory> {
474        let v = match (vector, self.embedder.as_ref()) {
475            (Some(v), _) => v,
476            (None, Some(e)) => e.embed(&[text])?.pop().ok_or_else(|| {
477                MnemoError::Invalid("essence: embedder returned no vector".into())
478            })?,
479            (None, None) => self.sentinel_vector(),
480        };
481        let mut m = Memory::new(text.to_string(), MemoryType::Semantic, v).with_importance(0.5);
482        m.metadata.insert(
483            MNEMO_META_KIND.into(),
484            Value::String(MNEMO_META_KIND_ESSENCE.into()),
485        );
486        m.metadata.insert(
487            MNEMO_META_SECTION.into(),
488            Value::String(section.as_str().into()),
489        );
490        if let EssenceScope::Thread(t) = &scope {
491            m.metadata
492                .insert(MNEMO_META_THREAD.into(), Value::String(t.clone()));
493        }
494        if let Some(r) = refs {
495            m.metadata.insert(
496                MNEMO_META_REFS.into(),
497                Value::Array(r.into_iter().map(Value::String).collect()),
498            );
499        }
500        if let Some(at) = status_at {
501            m.metadata.insert(
502                MNEMO_META_STATUS.into(),
503                Value::String(essence_status::OPEN.into()),
504            );
505            m.metadata.insert(
506                MNEMO_META_STATUS_HISTORY.into(),
507                Value::Array(vec![serde_json::json!({
508                    "status": essence_status::OPEN,
509                    "at": at,
510                })]),
511            );
512        }
513        Ok(m)
514    }
515
516    /// Axis-neutral unit vector (every component `1/sqrt(dims)`), used by
517    /// essence records without an embedder and by thread-link markers.
518    pub(super) fn sentinel_vector(&self) -> Vec<f32> {
519        let dims = self.dimensions.max(1);
520        vec![1.0 / (dims as f32).sqrt(); dims]
521    }
522
523    pub(super) fn load_essence_record(&mut self, id: &Ulid) -> Result<Memory> {
524        let m = self.get(id)?;
525        if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str()) != Some(MNEMO_META_KIND_ESSENCE)
526        {
527            return Err(MnemoError::Invalid(format!(
528                "essence: record {id} is not an essence record (missing `_mnemo.kind = essence`)"
529            )));
530        }
531        Ok(m)
532    }
533
534    pub(super) fn section_of(&self, m: &Memory) -> Result<EssenceSection> {
535        let s = m
536            .metadata
537            .get(MNEMO_META_SECTION)
538            .and_then(|v| v.as_str())
539            .ok_or_else(|| {
540                MnemoError::Invalid("essence: record missing `_mnemo.section`".into())
541            })?;
542        EssenceSection::parse(s)
543            .ok_or_else(|| MnemoError::Invalid(format!("essence: unknown section '{s}' on record")))
544    }
545
546    pub(super) fn essence_transition_status(
547        &mut self,
548        id: &Ulid,
549        to: &str,
550    ) -> Result<WriteOutcome> {
551        if self.read_only {
552            return Err(MnemoError::ReadOnly);
553        }
554        let existing = self.load_essence_record(id)?;
555        let section = self.section_of(&existing)?;
556        if !section.has_status_lifecycle() {
557            return Err(MnemoError::Invalid(format!(
558                "essence_transition: '{}' section has no status lifecycle",
559                section.as_str()
560            )));
561        }
562        let from = existing
563            .metadata
564            .get(MNEMO_META_STATUS)
565            .and_then(|v| v.as_str())
566            .unwrap_or(essence_status::OPEN);
567        if from == to {
568            return Ok(WriteOutcome::Unchanged);
569        }
570        let allowed = ESSENCE_TRANSITIONS
571            .iter()
572            .any(|(f, t)| *f == from && *t == to);
573        if !allowed {
574            return Err(MnemoError::Invalid(format!(
575                "essence_transition: '{from}' → '{to}' is not a valid transition for essence"
576            )));
577        }
578        // Mutate metadata in place and `put` back under the same id
579        // so `_mnemo.updated_at` gets stamped and access stats are
580        // preserved.
581        let mut new = existing.clone();
582        new.metadata
583            .insert(MNEMO_META_STATUS.into(), Value::String(to.to_string()));
584        let now = memory::now_secs();
585        let history_entry = serde_json::json!({ "status": to, "at": now });
586        let history = new
587            .metadata
588            .entry(MNEMO_META_STATUS_HISTORY.to_string())
589            .or_insert_with(|| Value::Array(Vec::new()));
590        if let Value::Array(arr) = history {
591            arr.push(history_entry);
592        }
593        new.metadata
594            .insert(MNEMO_META_UPDATED_AT.into(), Value::Number(now.into()));
595        new.id = existing.id;
596        new.created_at = existing.created_at;
597        self.put(new)?;
598        Ok(WriteOutcome::Updated)
599    }
600
601    // --- Phase 12.7c: continuation + commit_deltas ------------------
602}