Skip to main content

mnemo_engine/store/
essence.rs

1//! Phase 12.7b: essence items (add, append, update, resolve, remove) and the essence render.
2
3use super::*;
4
5/// Split `s` into chunks, each with `<= max_bytes` of UTF-8 content,
6/// landing on `char` boundaries. Preserves original ordering; a trailing
7/// chunk may be shorter than `max_bytes`.
8/// Phase 12.7b helper: pull the `_mnemo.status` string from an
9/// essence record. Returns `None` for timeline entries (they have no
10/// status) and for malformed records.
11pub(super) fn status_of(m: &Memory) -> Option<&str> {
12    m.metadata.get(MNEMO_META_STATUS).and_then(|v| v.as_str())
13}
14
15/// Phase 12.7b helper: render one essence item as a markdown bullet.
16/// Timeline entries get a date tag; section items get `[open]` /
17/// `[resolved]` so the reader can scan active work quickly.
18fn render_essence_item(section: EssenceSection, m: &Memory, now: i64) -> String {
19    if section == EssenceSection::Timeline {
20        let date = format_unix_date(m.created_at);
21        format!("- [{date}] {}\n", m.content)
22    } else {
23        let status = status_of(m).unwrap_or(essence_status::OPEN);
24        // 13a (13.6): an item past its review date, or its section's age.
25        let stale = crate::memory::stale_since(m, now)
26            .map(|t| format!(" [stale since {}]", format_unix_date(t)))
27            .unwrap_or_default();
28        format!("- [{status}] {}{stale}\n", m.content)
29    }
30}
31
32/// Minimal unix-seconds → `YYYY-MM-DD` formatter (no chrono/time
33/// dependency; mirrors the ISO 8601 parser in `transcript.rs`).
34pub(crate) fn format_unix_date(secs: i64) -> String {
35    let days_total = secs.div_euclid(86_400);
36    let (y, m, d) = civil_from_days(days_total);
37    format!("{y:04}-{m:02}-{d:02}")
38}
39
40/// Howard Hinnant's `civil_from_days` — inverse of `days_from_civil`,
41/// adequate for Gregorian dates since 1970. Pure arithmetic, no
42/// dependencies.
43fn civil_from_days(z: i64) -> (i64, u32, u32) {
44    let z = z + 719_468;
45    let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
46    let doe = (z - era * 146_097) as u64;
47    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
48    let y = yoe as i64 + era * 400;
49    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
50    let mp = (5 * doy + 2) / 153;
51    let d = (doy - (153 * mp + 2) / 5 + 1) as u32;
52    let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32;
53    let y = if m <= 2 { y + 1 } else { y };
54    (y, m, d)
55}
56
57impl Mnemo {
58    /// Phase 12.7b: add a new essence item to a section (any section
59    /// except [`EssenceSection::Timeline`]). Content ≤
60    /// [`memory::ESSENCE_SECTION_MAX_BYTES`] (280 bytes); items over
61    /// that are rejected. Starting status is `"open"`.
62    ///
63    /// Use [`Mnemo::essence_append_timeline`] for timeline entries —
64    /// they have a different cap (600 bytes), are append-only, and
65    /// carry no status lifecycle.
66    pub fn essence_add(
67        &mut self,
68        scope: EssenceScope,
69        section: EssenceSection,
70        text: &str,
71        refs: Option<Vec<String>>,
72    ) -> Result<Ulid> {
73        if self.read_only {
74            return Err(MnemoError::ReadOnly);
75        }
76        Self::check_essence_add(section, text)?;
77        self.essence_add_with(scope, section, text, refs, None)
78    }
79
80    /// [`Mnemo::essence_add`] with the item's vector supplied by the
81    /// caller instead of the registered embedder, so a caller can embed
82    /// `text` without holding the handle (the UniFFI binding does, as with
83    /// [`Mnemo::ingest_thread_with_vectors`]). Same validation, same record.
84    pub fn essence_add_with_vector(
85        &mut self,
86        scope: EssenceScope,
87        section: EssenceSection,
88        text: &str,
89        refs: Option<Vec<String>>,
90        vector: Vec<f32>,
91    ) -> Result<Ulid> {
92        if self.read_only {
93            return Err(MnemoError::ReadOnly);
94        }
95        Self::check_essence_add(section, text)?;
96        self.essence_add_with(scope, section, text, refs, Some(vector))
97    }
98
99    /// Validation shared by [`Mnemo::essence_add`] and `commit_deltas`.
100    pub(super) fn check_essence_add(section: EssenceSection, text: &str) -> Result<()> {
101        if section == EssenceSection::Timeline {
102            return Err(MnemoError::Invalid(
103                "essence_add: use essence_append_timeline for the timeline section".into(),
104            ));
105        }
106        if text.is_empty() {
107            return Err(MnemoError::Invalid(
108                "essence_add: text must not be empty".into(),
109            ));
110        }
111        if text.len() > section.item_max_bytes() {
112            return Err(MnemoError::Invalid(format!(
113                "essence_add: '{}' items must be <= {} bytes; got {}",
114                section.as_str(),
115                section.item_max_bytes(),
116                text.len()
117            )));
118        }
119        Ok(())
120    }
121
122    /// Write half of `essence_add`, with an optional precomputed vector.
123    pub(super) fn essence_add_with(
124        &mut self,
125        scope: EssenceScope,
126        section: EssenceSection,
127        text: &str,
128        refs: Option<Vec<String>>,
129        vector: Option<Vec<f32>>,
130    ) -> Result<Ulid> {
131        let now = memory::now_secs();
132        let mem = self.build_essence_record(scope, section, text, refs, Some(now), vector)?;
133        self.put(mem)
134    }
135
136    /// Phase 12.7b: append a new timeline entry (append-only
137    /// milestone). Content ≤ [`memory::ESSENCE_TIMELINE_MAX_BYTES`]
138    /// (600 bytes); entries over that are rejected (not truncated —
139    /// truncating a milestone silently loses the narrative).
140    pub fn essence_append_timeline(
141        &mut self,
142        scope: EssenceScope,
143        text: &str,
144        refs: Option<Vec<String>>,
145    ) -> Result<Ulid> {
146        if self.read_only {
147            return Err(MnemoError::ReadOnly);
148        }
149        Self::check_essence_append(text)?;
150        self.essence_append_with(scope, text, refs, None)
151    }
152
153    /// [`Mnemo::essence_append_timeline`] with a caller-supplied vector
154    /// (see [`Mnemo::essence_add_with_vector`]).
155    pub fn essence_append_timeline_with_vector(
156        &mut self,
157        scope: EssenceScope,
158        text: &str,
159        refs: Option<Vec<String>>,
160        vector: Vec<f32>,
161    ) -> Result<Ulid> {
162        if self.read_only {
163            return Err(MnemoError::ReadOnly);
164        }
165        Self::check_essence_append(text)?;
166        self.essence_append_with(scope, text, refs, Some(vector))
167    }
168
169    /// Validation shared by [`Mnemo::essence_append_timeline`] and
170    /// `commit_deltas`.
171    pub(super) fn check_essence_append(text: &str) -> Result<()> {
172        if text.is_empty() {
173            return Err(MnemoError::Invalid(
174                "essence_append_timeline: text must not be empty".into(),
175            ));
176        }
177        if text.len() > EssenceSection::Timeline.item_max_bytes() {
178            return Err(MnemoError::Invalid(format!(
179                "essence_append_timeline: entries must be <= {} bytes; got {}",
180                EssenceSection::Timeline.item_max_bytes(),
181                text.len()
182            )));
183        }
184        Ok(())
185    }
186
187    /// Write half of `essence_append_timeline`, with an optional
188    /// precomputed vector.
189    pub(super) fn essence_append_with(
190        &mut self,
191        scope: EssenceScope,
192        text: &str,
193        refs: Option<Vec<String>>,
194        vector: Option<Vec<f32>>,
195    ) -> Result<Ulid> {
196        let mem =
197            self.build_essence_record(scope, EssenceSection::Timeline, text, refs, None, vector)?;
198        self.put(mem)
199    }
200
201    /// Phase 12.7b: update an existing essence section item in place.
202    /// Returns [`WriteOutcome::Updated`] / [`WriteOutcome::Unchanged`].
203    /// Timeline entries are append-only — calling this on a timeline
204    /// record is an error.
205    pub fn essence_update(
206        &mut self,
207        id: &Ulid,
208        text: &str,
209        refs: Option<Vec<String>>,
210    ) -> Result<WriteOutcome> {
211        if self.read_only {
212            return Err(MnemoError::ReadOnly);
213        }
214        let existing = self.load_essence_record(id)?;
215        let section = self.section_of(&existing)?;
216        if section == EssenceSection::Timeline {
217            return Err(MnemoError::Invalid(
218                "essence_update: timeline entries are append-only; append a new one instead".into(),
219            ));
220        }
221        if text.is_empty() {
222            return Err(MnemoError::Invalid(
223                "essence_update: text must not be empty".into(),
224            ));
225        }
226        if text.len() > section.item_max_bytes() {
227            return Err(MnemoError::Invalid(format!(
228                "essence_update: '{}' items must be <= {} bytes; got {}",
229                section.as_str(),
230                section.item_max_bytes(),
231                text.len()
232            )));
233        }
234        let mut patch = MemoryPatch {
235            content: Some(text.to_string()),
236            ..Default::default()
237        };
238        // We don't have a usable embedder-free path for an in-place
239        // content update, so provide the same vector as the existing
240        // record (the text-vector drift guard in `update` will accept
241        // this).
242        patch.vector = Some(existing.vector.clone());
243        if let Some(r) = refs {
244            let mut meta = Map::new();
245            // Preserve the user-visible metadata (none for essence —
246            // everything lives under `_mnemo.*`) and let `update`'s
247            // metadata-replacement path keep the reserved keys.
248            meta.insert(
249                MNEMO_META_REFS.into(),
250                Value::Array(r.into_iter().map(Value::String).collect()),
251            );
252            patch.metadata = Some(meta);
253        }
254        self.update_unchecked(id, patch)
255    }
256
257    /// Phase 12.7b: mark a section item as resolved (open → resolved
258    /// transition). A new history entry lands on
259    /// `_mnemo.status_history`. Idempotent — resolving a record that
260    /// is already resolved is a no-op.
261    pub fn essence_resolve(&mut self, id: &Ulid) -> Result<WriteOutcome> {
262        self.essence_transition_status(id, essence_status::RESOLVED)
263    }
264
265    /// Phase 12.7b: soft-delete a section item. Same semantics as
266    /// [`Mnemo::delete`] on any record — space reclaimed by a later
267    /// compact.
268    pub fn essence_remove(&mut self, id: &Ulid) -> Result<()> {
269        if self.read_only {
270            return Err(MnemoError::ReadOnly);
271        }
272        self.delete(id)
273    }
274
275    /// Phase 12.7b: deterministic, budgeted markdown render of the
276    /// essence layer for a thread (or the project layer when
277    /// `scope = EssenceScope::Project`). `max_bytes` caps the output;
278    /// sections are emitted in the fixed [`EssenceSection::render_order`],
279    /// and each section's own budget is proportional to its position
280    /// in the order. Oldest timeline entries fold into a
281    /// `+N earlier milestones` tail when the budget runs out.
282    ///
283    /// The render always ends with an archive pointer so the reader
284    /// knows how to drill in deeper (`mnemo thread export …` or
285    /// `recall --thread …`).
286    pub fn essence(&mut self, scope: EssenceScope, max_bytes: usize) -> Result<String> {
287        let max_bytes = if max_bytes == 0 {
288            ESSENCE_DEFAULT_RENDER_BYTES
289        } else {
290            max_bytes
291        };
292        // Collect every live essence record for this scope.
293        // R-004.2: only the essence items, from the handle caches.
294        let entries = self.essence_entries()?;
295        let now = memory::now_secs();
296        let mut by_section: std::collections::BTreeMap<&'static str, Vec<Memory>> =
297            std::collections::BTreeMap::new();
298        for section in EssenceSection::render_order() {
299            by_section.insert(section.as_str(), Vec::new());
300        }
301        for e in &entries {
302            let m = self.read_memory(e)?;
303            if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
304                != Some(MNEMO_META_KIND_ESSENCE)
305            {
306                continue;
307            }
308            // Scope filter. Project-scope records are shown in every
309            // thread's render (and in the project render); thread-scope
310            // records only match when scope == same thread.
311            let rec_thread = m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str());
312            let matches_scope = match (&scope, rec_thread) {
313                (EssenceScope::Project, None) => true,
314                (EssenceScope::Thread(_), None) => true, // project-scope always shows
315                (EssenceScope::Thread(t), Some(rt)) => rt == t,
316                (EssenceScope::Project, Some(_)) => false,
317            };
318            if !matches_scope {
319                continue;
320            }
321            // 13a (13.5): the render shows public and team items.
322            if !crate::memory::VisibilitySet::DEFAULT.shows(&m) {
323                continue;
324            }
325            let section_str = m
326                .metadata
327                .get(MNEMO_META_SECTION)
328                .and_then(|v| v.as_str())
329                .unwrap_or("timeline");
330            by_section
331                .entry(
332                    EssenceSection::parse(section_str)
333                        .map(|s| s.as_str())
334                        .unwrap_or("timeline"),
335                )
336                .or_default()
337                .push(m);
338        }
339
340        // Sort each section's records:
341        // - Decisions + Timeline: newest-first (by created_at desc).
342        // - Everything else: sort open items before resolved, then
343        //   newest-first. Cheap, deterministic.
344        for (name, items) in by_section.iter_mut() {
345            let newest_first = matches!(*name, "decisions" | "timeline");
346            items.sort_by(|a, b| {
347                let a_resolved = status_of(a) == Some(essence_status::RESOLVED);
348                let b_resolved = status_of(b) == Some(essence_status::RESOLVED);
349                match (a_resolved, b_resolved) {
350                    (false, true) => std::cmp::Ordering::Less,
351                    (true, false) => std::cmp::Ordering::Greater,
352                    _ => {
353                        if newest_first {
354                            b.created_at.cmp(&a.created_at)
355                        } else {
356                            a.created_at.cmp(&b.created_at)
357                        }
358                    }
359                }
360            });
361        }
362
363        // Per-section budget: give intent / conventions / decisions a
364        // larger share (they frame the thread), timeline a smaller
365        // share (it can fold). This is deterministic because it
366        // depends only on `max_bytes`.
367        let header_reserve = 256usize; // header + archive pointer
368        let body_budget = max_bytes.saturating_sub(header_reserve);
369        let per_section_budget = |section: EssenceSection| -> usize {
370            match section {
371                EssenceSection::Intent => body_budget / 8,
372                EssenceSection::Conventions => body_budget / 6,
373                EssenceSection::Decisions => body_budget / 4,
374                EssenceSection::State => body_budget / 8,
375                EssenceSection::Open => body_budget / 8,
376                EssenceSection::Next => body_budget / 12,
377                EssenceSection::Lessons => body_budget / 12,
378                EssenceSection::Timeline => body_budget / 4,
379            }
380        };
381
382        let mut out = String::with_capacity(max_bytes);
383        let scope_label = match &scope {
384            EssenceScope::Thread(t) => format!("thread `{t}`"),
385            EssenceScope::Project => "project".to_string(),
386        };
387        out.push_str(&format!("# essence — {scope_label}\n\n"));
388
389        // Phase 12.7c: continuation. If this is a thread render AND
390        // the thread has a `_mnemo.continues` link, pull the prior
391        // thread's open section items, its `next`, and its last-N
392        // timeline entries in ahead of the new thread's own essence.
393        if let EssenceScope::Thread(new_id) = &scope {
394            if let Some(prev_id) = self.predecessor_thread(new_id)? {
395                let continuation = self
396                    .render_continuation_prelude(&prev_id, max_bytes.saturating_sub(out.len()))?;
397                if !continuation.is_empty() && out.len() + continuation.len() <= max_bytes {
398                    out.push_str(&continuation);
399                }
400            }
401        }
402
403        for section in EssenceSection::render_order() {
404            let items = match by_section.get(section.as_str()) {
405                Some(v) if !v.is_empty() => v,
406                _ => continue,
407            };
408            let budget = per_section_budget(*section);
409            let section_header = format!("## {}\n", section.as_str());
410            if out.len() + section_header.len() > max_bytes {
411                break;
412            }
413            out.push_str(&section_header);
414            let mut used = 0usize;
415            let mut rendered = 0usize;
416            for item in items {
417                let line = render_essence_item(*section, item, now);
418                if used + line.len() > budget {
419                    break;
420                }
421                if out.len() + line.len() > max_bytes {
422                    break;
423                }
424                out.push_str(&line);
425                used += line.len();
426                rendered += 1;
427            }
428            // Fold the remainder for timeline.
429            if *section == EssenceSection::Timeline && rendered < items.len() {
430                let fold = format!("- +{} earlier milestones\n", items.len() - rendered);
431                if out.len() + fold.len() <= max_bytes {
432                    out.push_str(&fold);
433                }
434            }
435            if out.len() < max_bytes {
436                out.push('\n');
437            }
438        }
439
440        // Archive pointer.
441        let pointer = match &scope {
442            EssenceScope::Thread(t) => format!(
443                "> full history: `mnemo thread export <file> --thread {t}` or `recall --thread {t}`\n"
444            ),
445            EssenceScope::Project => String::from(
446                "> full history: `mnemo thread export <file> --thread <id>` or `recall --thread <id>`\n",
447            ),
448        };
449        if out.len() + pointer.len() <= max_bytes {
450            out.push_str(&pointer);
451        }
452        Ok(out)
453    }
454
455    // --- essence internal helpers ----------------------------------
456
457    /// Build an essence record. `vector` is a precomputed embedding (the
458    /// 12.7d `commit_deltas` batch path); `None` embeds `text` with the
459    /// registered embedder, or falls back to an axis-neutral unit vector
460    /// when there is none. Essence records are read by `_mnemo.kind` and
461    /// scope, not by similarity, so the fallback costs no retrieval
462    /// quality; it only has to pass `put`'s non-zero guard.
463    pub(super) fn build_essence_record(
464        &self,
465        scope: EssenceScope,
466        section: EssenceSection,
467        text: &str,
468        refs: Option<Vec<String>>,
469        status_at: Option<i64>,
470        vector: Option<Vec<f32>>,
471    ) -> Result<Memory> {
472        let v = match (vector, self.embedder.as_ref()) {
473            (Some(v), _) => v,
474            (None, Some(e)) => e.embed(&[text])?.pop().ok_or_else(|| {
475                MnemoError::Invalid("essence: embedder returned no vector".into())
476            })?,
477            (None, None) => self.sentinel_vector(),
478        };
479        let mut m = Memory::new(text.to_string(), MemoryType::Semantic, v).with_importance(0.5);
480        m.metadata.insert(
481            MNEMO_META_KIND.into(),
482            Value::String(MNEMO_META_KIND_ESSENCE.into()),
483        );
484        m.metadata.insert(
485            MNEMO_META_SECTION.into(),
486            Value::String(section.as_str().into()),
487        );
488        if let EssenceScope::Thread(t) = &scope {
489            m.metadata
490                .insert(MNEMO_META_THREAD.into(), Value::String(t.clone()));
491        }
492        if let Some(r) = refs {
493            m.metadata.insert(
494                MNEMO_META_REFS.into(),
495                Value::Array(r.into_iter().map(Value::String).collect()),
496            );
497        }
498        if let Some(at) = status_at {
499            m.metadata.insert(
500                MNEMO_META_STATUS.into(),
501                Value::String(essence_status::OPEN.into()),
502            );
503            m.metadata.insert(
504                MNEMO_META_STATUS_HISTORY.into(),
505                Value::Array(vec![serde_json::json!({
506                    "status": essence_status::OPEN,
507                    "at": at,
508                })]),
509            );
510        }
511        Ok(m)
512    }
513
514    /// Axis-neutral unit vector (every component `1/sqrt(dims)`), used by
515    /// essence records without an embedder and by thread-link markers.
516    pub(super) fn sentinel_vector(&self) -> Vec<f32> {
517        let dims = self.dimensions.max(1);
518        vec![1.0 / (dims as f32).sqrt(); dims]
519    }
520
521    pub(super) fn load_essence_record(&mut self, id: &Ulid) -> Result<Memory> {
522        let m = self.get(id)?;
523        if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str()) != Some(MNEMO_META_KIND_ESSENCE)
524        {
525            return Err(MnemoError::Invalid(format!(
526                "essence: record {id} is not an essence record (missing `_mnemo.kind = essence`)"
527            )));
528        }
529        Ok(m)
530    }
531
532    pub(super) fn section_of(&self, m: &Memory) -> Result<EssenceSection> {
533        let s = m
534            .metadata
535            .get(MNEMO_META_SECTION)
536            .and_then(|v| v.as_str())
537            .ok_or_else(|| {
538                MnemoError::Invalid("essence: record missing `_mnemo.section`".into())
539            })?;
540        EssenceSection::parse(s)
541            .ok_or_else(|| MnemoError::Invalid(format!("essence: unknown section '{s}' on record")))
542    }
543
544    pub(super) fn essence_transition_status(
545        &mut self,
546        id: &Ulid,
547        to: &str,
548    ) -> Result<WriteOutcome> {
549        if self.read_only {
550            return Err(MnemoError::ReadOnly);
551        }
552        let existing = self.load_essence_record(id)?;
553        let section = self.section_of(&existing)?;
554        if !section.has_status_lifecycle() {
555            return Err(MnemoError::Invalid(format!(
556                "essence_transition: '{}' section has no status lifecycle",
557                section.as_str()
558            )));
559        }
560        let from = existing
561            .metadata
562            .get(MNEMO_META_STATUS)
563            .and_then(|v| v.as_str())
564            .unwrap_or(essence_status::OPEN);
565        if from == to {
566            return Ok(WriteOutcome::Unchanged);
567        }
568        let allowed = ESSENCE_TRANSITIONS
569            .iter()
570            .any(|(f, t)| *f == from && *t == to);
571        if !allowed {
572            return Err(MnemoError::Invalid(format!(
573                "essence_transition: '{from}' → '{to}' is not a valid transition for essence"
574            )));
575        }
576        // Mutate metadata in place and `put` back under the same id
577        // so `_mnemo.updated_at` gets stamped and access stats are
578        // preserved.
579        let mut new = existing.clone();
580        new.metadata
581            .insert(MNEMO_META_STATUS.into(), Value::String(to.to_string()));
582        let now = memory::now_secs();
583        let history_entry = serde_json::json!({ "status": to, "at": now });
584        let history = new
585            .metadata
586            .entry(MNEMO_META_STATUS_HISTORY.to_string())
587            .or_insert_with(|| Value::Array(Vec::new()));
588        if let Value::Array(arr) = history {
589            arr.push(history_entry);
590        }
591        new.metadata
592            .insert(MNEMO_META_UPDATED_AT.into(), Value::Number(now.into()));
593        new.id = existing.id;
594        new.created_at = existing.created_at;
595        self.put(new)?;
596        Ok(WriteOutcome::Updated)
597    }
598
599    // --- Phase 12.7c: continuation + commit_deltas ------------------
600}