Skip to main content

mnemo_engine/store/
crud.rs

1//! Public CRUD (remember, get, delete, embedder), and the handle caches (key slots, superseded set, thread layers).
2
3use super::*;
4
5/// R-004.2: where the thread layers and the onboarding briefing live,
6/// built by the handle's one lazy scan (`ensure_key_caches`, together with
7/// the 12.1 key caches) and then kept current by `put` and `delete`. It
8/// lets `about`, `essence`, `ingest_thread`, `thread_export` and
9/// thread-pinned recall read just their own records instead of decrypting
10/// the whole file each time.
11#[derive(Debug, Default)]
12pub(super) struct LayerCache {
13    /// Live essence item ids.
14    pub(super) essence: std::collections::HashSet<u128>,
15    /// Live ids tagged `metadata.area = "onboarding"` (superseded versions
16    /// included; `about` drops those when it reads them).
17    pub(super) onboarding: std::collections::HashSet<u128>,
18    /// Per `_mnemo.thread`: its records and archive state.
19    pub(super) threads: std::collections::HashMap<String, ThreadCache>,
20    /// Thread → (the thread it continues, the link record's id).
21    pub(super) links: std::collections::HashMap<String, (String, u128)>,
22    /// FMT-9e: on v9, thread-turn records the lazy scan didn't decode,
23    /// by their row's `thread_hash`. A thread's bucket is decoded into
24    /// `threads` the first time that thread is used
25    /// ([`Mnemo::warm_thread`]), so a cold handle never decodes (or
26    /// decompresses) the turns of threads it doesn't touch.
27    pub(super) cold_turns: std::collections::HashMap<u32, Vec<u128>>,
28}
29
30/// One thread's records (turns, essence items, link), the turn ids already
31/// archived (with how many records each has: long turns are chunked), and
32/// the highest `_mnemo.seq` seen.
33#[derive(Debug, Default)]
34pub(super) struct ThreadCache {
35    pub(super) records: std::collections::HashSet<u128>,
36    pub(super) turns: std::collections::HashMap<String, u32>,
37    pub(super) max_seq: Option<u64>,
38}
39
40pub(super) fn meta_str<'a>(m: &'a Memory, key: &str) -> Option<&'a str> {
41    m.metadata.get(key).and_then(|v| v.as_str())
42}
43
44pub(super) fn is_onboarding(m: &Memory) -> bool {
45    meta_str(m, "area").is_some_and(|s| s.eq_ignore_ascii_case("onboarding"))
46}
47
48impl LayerCache {
49    pub(super) fn add(&mut self, m: &Memory) {
50        let id = m.id.0;
51        let kind = meta_str(m, MNEMO_META_KIND);
52        if kind == Some(MNEMO_META_KIND_ESSENCE) {
53            self.essence.insert(id);
54        }
55        if is_onboarding(m) {
56            self.onboarding.insert(id);
57        }
58        let Some(thread) = meta_str(m, MNEMO_META_THREAD) else {
59            return;
60        };
61        let tc = self.threads.entry(thread.to_string()).or_default();
62        tc.records.insert(id);
63        if kind == Some(MNEMO_META_KIND_TURN) {
64            if let Some(turn_id) = meta_str(m, MNEMO_META_TURN_ID) {
65                *tc.turns.entry(turn_id.to_string()).or_insert(0) += 1;
66            }
67            if let Some(seq) = m.metadata.get(MNEMO_META_SEQ).and_then(|v| v.as_u64()) {
68                tc.max_seq = Some(tc.max_seq.map_or(seq, |s| s.max(seq)));
69            }
70        }
71        if kind == Some(MNEMO_META_KIND_THREAD_LINK) {
72            if let Some(from) = meta_str(m, MNEMO_META_CONTINUES) {
73                self.links
74                    .entry(thread.to_string())
75                    .or_insert_with(|| (from.to_string(), id));
76            }
77        }
78    }
79
80    /// Undo [`LayerCache::add`] for the stored version of a record that is
81    /// being overwritten or deleted. A thread with no records left is
82    /// dropped, so a forgotten thread's `_mnemo.seq` restarts at 0. Partial
83    /// deletes keep the old maximum (sequence gaps are harmless).
84    pub(super) fn remove(&mut self, m: &Memory) {
85        let id = m.id.0;
86        self.essence.remove(&id);
87        self.onboarding.remove(&id);
88        let Some(thread) = meta_str(m, MNEMO_META_THREAD) else {
89            return;
90        };
91        if self.links.get(thread).is_some_and(|(_, link)| *link == id) {
92            self.links.remove(thread);
93        }
94        let Some(tc) = self.threads.get_mut(thread) else {
95            return;
96        };
97        tc.records.remove(&id);
98        if meta_str(m, MNEMO_META_KIND) == Some(MNEMO_META_KIND_TURN) {
99            if let Some(turn_id) = meta_str(m, MNEMO_META_TURN_ID) {
100                if let Some(n) = tc.turns.get_mut(turn_id) {
101                    *n -= 1;
102                    if *n == 0 {
103                        tc.turns.remove(turn_id);
104                    }
105                }
106            }
107        }
108        if tc.records.is_empty() {
109            self.threads.remove(thread);
110        }
111    }
112}
113
114impl Mnemo {
115    /// Register a text embedder for this handle. Enables
116    /// [`Mnemo::remember_text`] and the text-side `RecallRequest`
117    /// convenience path. Does two checks before storing the embedder:
118    ///
119    /// 1. **Dimension check.** The embedder's `identity().dimensions`
120    ///    must equal [`Mnemo::dimensions`] (the width this database
121    ///    was created with). Mismatch returns
122    ///    [`MnemoError::DimensionMismatch`].
123    /// 2. **Identity check (EMBED-ID-1).** Every file records the
124    ///    identity (name, dimensions, normalize) of the first embedder
125    ///    attached to it, in a reserved, encrypted record that isn't a
126    ///    memory: `len`, `count`, `stats`, recall, `about` and `get`
127    ///    never see it, and compaction, the rekeyed exports and the
128    ///    passphrase import carry it. After that, any other identity,
129    ///    even at the same width (another model, or another revision of
130    ///    the same one), returns [`MnemoError::EmbedderMismatch`]: a
131    ///    store is bound to the model that first wrote to it, so a new
132    ///    model needs a new store.
133    ///
134    /// **Files with a manifest** (`mnemo init`, `seed.json`) also carry
135    /// the identity on the manifest at `metadata.embedder = { name,
136    /// dimensions, normalize }` (the legacy flat `embedder_name` /
137    /// `embedder_dimensions` / `embedder_normalize` keys are read as an
138    /// alias). The first registration upserts it there too, as before.
139    /// If the manifest and the record disagree, this returns
140    /// [`MnemoError::Invalid`] naming both.
141    ///
142    /// **Files from before EMBED-ID-1** have no record: the first
143    /// embedder attached after the upgrade is recorded. The check
144    /// protects from then on; it can't tell which model made the older
145    /// vectors.
146    ///
147    /// **Read-only handles.** `set_embedder` on an `open_read_only`
148    /// handle performs both checks and adopts the embedder in memory;
149    /// it never writes. Readers (CLI `--text`, agent-side `recall_text`)
150    /// can therefore use the text path next to a writer that holds
151    /// the exclusive lock.
152    ///
153    /// On every failure path the handle's embedder slot is left
154    /// unchanged. On success the embedder is wrapped in an `Arc<dyn
155    /// Embedder>` and consulted by every text-side call made through
156    /// this handle. The slot is per-handle and not persisted — a
157    /// fresh open must re-register (same identity, so the check
158    /// round-trips cleanly).
159    pub fn set_embedder<E: Embedder + 'static>(&mut self, embedder: E) -> Result<()> {
160        let identity = embedder.identity();
161        if identity.dimensions != self.dimensions {
162            return Err(MnemoError::DimensionMismatch {
163                expected: self.dimensions,
164                got: identity.dimensions,
165            });
166        }
167
168        let (onboarding, on_manifest, upsert_target, on_record) = self.embedder_records()?;
169
170        if let (Some(manifest), Some(record)) = (&on_manifest, &on_record) {
171            if manifest != record {
172                return Err(embed::identity_conflict_err(manifest, record));
173            }
174        }
175        if let Some(stored) = on_record.as_ref().or(on_manifest.as_ref()) {
176            if *stored != identity {
177                return Err(embed::identity_mismatch_err(stored, &identity));
178            }
179        }
180
181        // Record what's missing. A read-only handle writes nothing, but
182        // still adopts the embedder in memory.
183        if !self.read_only {
184            let mut wrote = false;
185            if on_manifest.is_none() {
186                if let Some(idx) = upsert_target {
187                    let mut m = onboarding[idx].clone();
188                    embed::write_identity_into_metadata(&mut m.metadata, &identity);
189                    // Reuse the memory's existing id so this is an in-place
190                    // update of the scaffold manifest, not a new entry.
191                    self.put(m)?;
192                    wrote = true;
193                }
194            }
195            if on_record.is_none() {
196                self.record_identity(&identity)?;
197                wrote = true;
198            }
199            if wrote {
200                self.flush()?;
201            }
202        }
203
204        self.embedder = Some(std::sync::Arc::new(embedder));
205        Ok(())
206    }
207
208    /// Where the file records an embedder identity (EMBED-ID-1): the
209    /// onboarding memories, the manifest's identity if it has one, the first
210    /// onboarding memory without one (the upsert target), and the reserved
211    /// record's identity.
212    #[allow(clippy::type_complexity)]
213    fn embedder_records(
214        &mut self,
215    ) -> Result<(
216        Vec<Memory>,
217        Option<embed::EmbedderIdentity>,
218        Option<usize>,
219        Option<embed::EmbedderIdentity>,
220    )> {
221        let onboarding = self.about_in(crate::memory::VisibilitySet::ALL)?;
222        let mut on_manifest: Option<embed::EmbedderIdentity> = None;
223        let mut upsert_target: Option<usize> = None;
224        for (i, m) in onboarding.iter().enumerate() {
225            if let Some(saw) = embed::read_identity_from_metadata(&m.metadata) {
226                on_manifest = Some(saw);
227                break;
228            }
229            if upsert_target.is_none() {
230                upsert_target = Some(i);
231            }
232        }
233        let on_record = self.recorded_identity()?;
234        Ok((onboarding, on_manifest, upsert_target, on_record))
235    }
236
237    /// Whether [`Mnemo::set_embedder`] with this identity would write to the
238    /// file (to record the identity where it's missing). A shared handle
239    /// attaches an already recorded embedder in a read window (RB-007.6).
240    pub(super) fn embedder_would_record(
241        &mut self,
242        identity: &embed::EmbedderIdentity,
243    ) -> Result<bool> {
244        if identity.dimensions != self.dimensions {
245            // `set_embedder` refuses it without writing.
246            return Ok(false);
247        }
248        let (_, on_manifest, upsert_target, on_record) = self.embedder_records()?;
249        Ok((on_manifest.is_none() && upsert_target.is_some()) || on_record.is_none())
250    }
251
252    /// The embedder identity this file records (EMBED-ID-1), or `None`
253    /// before the first `set_embedder` on a read-write handle. Reads the
254    /// reserved record without touching any counter.
255    pub fn embedder_identity(&mut self) -> Result<Option<embed::EmbedderIdentity>> {
256        self.recorded_identity()
257    }
258
259    /// Borrow the currently-registered embedder, if any. Returns
260    /// `None` when no embedder has been set via
261    /// [`Mnemo::set_embedder`]. Bindings use this to drive their own
262    /// text-side convenience methods without duplicating the identity
263    /// check.
264    pub fn embedder(&self) -> Option<&dyn Embedder> {
265        self.embedder.as_deref()
266    }
267
268    /// Shared handle to the registered embedder, cloned so the caller
269    /// can drop the borrow on this `Mnemo` before invoking `embed()`.
270    /// FIX-1 / PY-1 (R-001.11): the Python binding needs this so an
271    /// embedder callback that re-enters `Mnemo` (e.g. a Python
272    /// embedder that calls `db.dimensions()` or `db.cache_get(...)`)
273    /// doesn't hit PyO3's "Already borrowed" — we clone the Arc under
274    /// a short `&self` borrow, release that borrow, then invoke
275    /// `embed()` with no `&mut Mnemo` held.
276    pub fn embedder_arc(&self) -> Option<std::sync::Arc<dyn Embedder>> {
277        self.embedder.clone()
278    }
279
280    /// Embed `content` with the registered embedder and `remember` the
281    /// result. Returns the new memory's ULID. Convenience path for
282    /// callers who have a string, want it stored, and don't need any
283    /// of the per-memory options [`Memory`] exposes (agent, importance,
284    /// metadata, ttl, scope) — build a [`Memory`] by hand and call
285    /// [`Mnemo::remember`] for those.
286    ///
287    /// Fails with [`MnemoError::Invalid`] if no embedder has been
288    /// registered on this handle (call [`Mnemo::set_embedder`] first);
289    /// with [`MnemoError::DimensionMismatch`] if the embedder returned
290    /// the wrong width for this database.
291    pub fn remember_text(&mut self, content: &str, memory_type: MemoryType) -> Result<Ulid> {
292        if self.read_only {
293            return Err(MnemoError::ReadOnly);
294        }
295        let embedder = self.embedder.as_ref().ok_or_else(|| {
296            MnemoError::Invalid(
297                "no embedder registered on this handle; call set_embedder first".into(),
298            )
299        })?;
300        let mut vectors = embedder.embed(&[content])?;
301        let vector = vectors.pop().ok_or_else(|| {
302            MnemoError::Invalid("embedder returned no vector for the input text".into())
303        })?;
304        if vector.len() != self.dimensions {
305            return Err(MnemoError::DimensionMismatch {
306                expected: self.dimensions,
307                got: vector.len(),
308            });
309        }
310        let memory = Memory::new(content.to_string(), memory_type, vector);
311        self.remember(memory)
312    }
313
314    /// Store a memory. Returns its ULID. If the memory's ID is nil a fresh
315    /// one is assigned; an existing ID overwrites in place.
316    ///
317    /// Metadata keys starting with `_mnemo.` are the engine's (supersession,
318    /// keys, threads, kinds): a new memory carrying one is refused with
319    /// [`MnemoError::Invalid`]. An in-place overwrite may carry the stored
320    /// record's own `_mnemo.*` keys through unchanged (the usual `get`,
321    /// change a field, `remember`), but can't add, change or drop one.
322    pub fn remember(&mut self, memory: Memory) -> Result<Ulid> {
323        if self.read_only {
324            return Err(MnemoError::ReadOnly);
325        }
326        self.check_caller_metadata(&memory)?;
327        self.put(memory)
328    }
329
330    /// RB-001.24: the caller's metadata may not set engine state. For a
331    /// memory with a nil or unknown id, any `_mnemo.*` key is refused; for
332    /// an overwrite in place, the `_mnemo.*` keys must equal the stored
333    /// record's.
334    pub(super) fn check_caller_metadata(&mut self, memory: &Memory) -> Result<()> {
335        // 13a: visibility and freshness are the caller's to set (checked);
336        // every other `_mnemo.*` key is the engine's.
337        for (k, v) in memory.metadata.iter().filter(|(k, _)| is_caller_key(k)) {
338            crate::memory::check_caller_key(k, v).map_err(MnemoError::Invalid)?;
339        }
340        let reserved: Vec<(&String, &Value)> = memory
341            .metadata
342            .iter()
343            .filter(|(k, _)| k.starts_with(crate::memory::MNEMO_META_NS) && !is_caller_key(k))
344            .collect();
345        if reserved.is_empty() {
346            return Ok(());
347        }
348        let stored = if memory.id.is_nil() {
349            None
350        } else {
351            match self.index.get(memory.id.0, &self.catalog) {
352                Some(idx) if !self.catalog[idx].deleted => {
353                    let entry = self.catalog[idx].clone();
354                    Some(self.read_memory(&entry)?)
355                }
356                _ => None,
357            }
358        };
359        if let Some(old) = stored {
360            let old_reserved: Vec<(&String, &Value)> = old
361                .metadata
362                .iter()
363                .filter(|(k, _)| k.starts_with(crate::memory::MNEMO_META_NS) && !is_caller_key(k))
364                .collect();
365            if old_reserved == reserved {
366                return Ok(());
367            }
368        }
369        Err(reserved_key_error(reserved[0].0))
370    }
371
372    /// Fetch a memory by ID.
373    pub fn get(&mut self, id: &Ulid) -> Result<Memory> {
374        let idx = self
375            .index
376            .get(id.0, &self.catalog)
377            .ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
378        let entry = self.catalog[idx].clone();
379        if entry.deleted {
380            return Err(MnemoError::NotFound(id.to_string()));
381        }
382        self.read_memory(&entry)
383    }
384
385    /// Soft-delete a memory (tombstoned; space reclaimed by `compact`).
386    ///
387    /// A review finding is refused (RB-014): a wrong one is `rejected`
388    /// with [`Mnemo::finding_triage`], which keeps its history.
389    pub fn delete(&mut self, id: &Ulid) -> Result<()> {
390        if self.read_only {
391            return Err(MnemoError::ReadOnly);
392        }
393        let idx = self
394            .index
395            .get(id.0, &self.catalog)
396            .ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
397        // R-004.2 / 12.1: with the handle caches built, read the record
398        // being deleted (one decrypt) so the key slot, superseded set and
399        // thread layers drop exactly what it contributed.
400        let entry = self.catalog[idx].clone();
401        // RB-014: a finding's row is a coordination row, or a fact row when
402        // an older build compacted the file; only those are read for it.
403        let maybe_finding = entry.v9.is_none_or(|r| {
404            r.kind == super::records::ROW_KIND_COORDINATION
405                || r.kind == super::records::ROW_KIND_FACT
406        });
407        if !entry.deleted && (self.layers.is_some() || maybe_finding) {
408            let old = self.read_memory(&entry)?;
409            if super::findings::is_finding(&old) {
410                return Err(super::findings::finding_write_refused("deleted"));
411            }
412            if self.layers.is_some() {
413                self.note_removed(&old)?;
414            }
415        }
416        if !self.catalog[idx].deleted {
417            self.catalog[idx].deleted = true;
418            self.dirty_catalog = true;
419            self.touch(idx);
420            // FMT-9f: drop its document from the stats in place.
421            if let (Some(stats), Some(r)) = (self.bm25.stats.as_mut(), entry.v9) {
422                if r.doc != 0 {
423                    stats.remove(r.doc);
424                }
425            }
426            if self.is_v9() {
427                self.kill_record(idx, &entry);
428                self.kill_vector(idx, &entry);
429            } else {
430                self.release_pages(entry.start_page, entry.page_count as u64);
431            }
432        }
433        if let Some(ann) = &mut self.ann {
434            if ann.contains(id.0) {
435                ann.remove(id.0);
436                self.dirty_index = true;
437                self.index_persist.changes += 1;
438            }
439        }
440        // FIX-1 R-001.8 part 2: we can't tell whether this delete
441        // removed the last memory of its agent without a decrypt
442        // (the catalog entry doesn't carry `agent_id` in format v8),
443        // so just invalidate the lazy set. The next `stats()` call
444        // rebuilds from live memories.
445        self.agent_set = None;
446        // Phase 12.1: delete of a keyed record removes it from the slot,
447        // and delete of a superseded record frees a `history` entry. When
448        // the caches are built, `note_removed` above did that exactly;
449        // otherwise they're built from scratch on first use anyway.
450        Ok(())
451    }
452
453    /// Keep the handle caches (12.1 key slots, superseded set, R-004.2
454    /// layers) current after `put` writes `m`. No-op until they're built.
455    /// FMT-9e: a cold thread is warmed first, so its layer entry is whole.
456    pub(super) fn note_added(&mut self, m: &Memory) -> Result<()> {
457        if let Some(thread) = meta_str(m, MNEMO_META_THREAD) {
458            self.warm_thread(thread)?;
459        }
460        let id = m.id.0;
461        if let (Some(keys), Some(sup)) = (self.key_index.as_mut(), self.superseded.as_mut()) {
462            if m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) {
463                sup.insert(id);
464            } else if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
465                keys.insert(Self::key_slot_for(m, k), id);
466            }
467        }
468        if let Some(layers) = self.layers.as_mut() {
469            layers.add(m);
470        }
471        Ok(())
472    }
473
474    /// Undo [`Mnemo::note_added`] for the stored version of a record that's
475    /// about to be overwritten or deleted.
476    pub(super) fn note_removed(&mut self, m: &Memory) -> Result<()> {
477        if let Some(thread) = meta_str(m, MNEMO_META_THREAD) {
478            self.warm_thread(thread)?;
479        }
480        let id = m.id.0;
481        if let (Some(keys), Some(sup)) = (self.key_index.as_mut(), self.superseded.as_mut()) {
482            sup.remove(&id);
483            if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
484                let slot = Self::key_slot_for(m, k);
485                if keys.get(&slot) == Some(&id) {
486                    keys.remove(&slot);
487                }
488            }
489        }
490        if let Some(layers) = self.layers.as_mut() {
491            layers.remove(m);
492        }
493        Ok(())
494    }
495
496    /// FMT-9e: decode `thread`'s cold turn records (see
497    /// [`LayerCache::cold_turns`]) into the layer cache. Every record in
498    /// the thread's hash bucket is decoded and filed under its own thread,
499    /// so a hash collision just warms both threads. A no-op when the
500    /// caches aren't built or the bucket is already warm.
501    pub(super) fn warm_thread(&mut self, thread: &str) -> Result<()> {
502        self.warm_thread_decoded(thread).map(|_| ())
503    }
504
505    /// [`Mnemo::warm_thread`], returning the records it decoded (without
506    /// vectors), so a caller about to read the thread decodes each once.
507    pub(super) fn warm_thread_decoded(
508        &mut self,
509        thread: &str,
510    ) -> Result<std::collections::HashMap<u128, Memory>> {
511        let mut decoded = std::collections::HashMap::new();
512        let h = fnv1a32(&[thread.as_bytes()]);
513        let Some(ids) = self.layers.as_mut().and_then(|l| l.cold_turns.remove(&h)) else {
514            return Ok(decoded);
515        };
516        for e in self.live_entries(ids) {
517            let m = self.read_memory_meta(&e)?;
518            if let Some(layers) = self.layers.as_mut() {
519                layers.add(&m);
520            }
521            decoded.insert(e.id, m);
522        }
523        Ok(decoded)
524    }
525
526    /// Decode a record's memory without its vector or live access stats:
527    /// what the handle caches need (metadata, scope, agent), without the
528    /// v9 vector-column read [`Mnemo::read_memory`] adds.
529    pub(super) fn read_memory_meta(&mut self, e: &CatalogEntry) -> Result<Memory> {
530        let bytes = self.read_record(e)?;
531        self.decode_memory(&bytes)
532    }
533
534    /// The live catalog entries for `ids` (ids that are gone are skipped).
535    pub(super) fn live_entries(&self, ids: impl IntoIterator<Item = u128>) -> Vec<CatalogEntry> {
536        ids.into_iter()
537            .filter_map(|id| {
538                self.index
539                    .get(id, &self.catalog)
540                    .map(|i| self.catalog[i].clone())
541            })
542            .filter(|e| !e.deleted)
543            .collect()
544    }
545
546    /// Catalog entries of the essence items, from the handle caches.
547    pub(super) fn essence_entries(&mut self) -> Result<Vec<CatalogEntry>> {
548        self.ensure_key_caches()?;
549        let ids: Vec<u128> = self
550            .layers
551            .as_ref()
552            .map(|l| l.essence.iter().copied().collect())
553            .unwrap_or_default();
554        Ok(self.live_entries(ids))
555    }
556
557    /// Catalog entries of every record filed under `thread`, from the
558    /// handle caches.
559    pub(super) fn thread_entries(&mut self, thread: &str) -> Result<Vec<CatalogEntry>> {
560        Ok(self.thread_entries_decoded(thread)?.0)
561    }
562
563    /// [`Mnemo::thread_entries`], plus the records that warming a cold
564    /// thread just decoded (see [`Mnemo::warm_thread_decoded`]).
565    pub(super) fn thread_entries_decoded(
566        &mut self,
567        thread: &str,
568    ) -> Result<(Vec<CatalogEntry>, std::collections::HashMap<u128, Memory>)> {
569        self.ensure_key_caches()?;
570        let decoded = self.warm_thread_decoded(thread)?;
571        let ids: Vec<u128> = self
572            .layers
573            .as_ref()
574            .and_then(|l| l.threads.get(thread))
575            .map(|tc| tc.records.iter().copied().collect())
576            .unwrap_or_default();
577        Ok((self.live_entries(ids), decoded))
578    }
579
580    // --- Phase 12.1: edit in place + keyed supersession -----------------
581
582    /// Scope-aware key-slot identifier for `memory` and a caller key.
583    ///
584    /// * [`Scope::Private`] → owned by the memory's `agent_id`.
585    /// * [`Scope::Shared`]  → one slot across the whole file.
586    pub(super) fn key_slot_for(memory: &Memory, key: &str) -> crate::store::KeySlot {
587        let scope_owner = match memory.scope {
588            Scope::Private => Some(memory.agent_id.clone()),
589            Scope::Shared => None,
590        };
591        crate::store::KeySlot {
592            scope_owner,
593            key: key.to_string(),
594        }
595    }
596
597    /// The handle's one lazy scan: builds the 12.1 `key_index` +
598    /// `superseded` caches and the R-004.2 layer cache (essence ids,
599    /// onboarding ids, threads, links) from live records, and prunes
600    /// non-fact ids from a loaded ANN index (files written before v0.10.0
601    /// indexed every kind). Decrypts every live record once per handle;
602    /// `put` and `delete` keep the result current after that.
603    pub(super) fn ensure_key_caches(&mut self) -> Result<()> {
604        if self.key_index.is_some() && self.superseded.is_some() && self.layers.is_some() {
605            return Ok(());
606        }
607        let entries: Vec<CatalogEntry> = self
608            .catalog
609            .iter()
610            .filter(|e| !e.deleted)
611            .cloned()
612            .collect();
613        let mut keys: std::collections::HashMap<crate::store::KeySlot, u128> =
614            std::collections::HashMap::new();
615        let mut superseded: std::collections::HashSet<u128> = std::collections::HashSet::new();
616        let mut layers = LayerCache::default();
617        let mut non_facts: Vec<u128> = Vec::new();
618        for e in &entries {
619            // FMT-9e: a v9 turn row with no key slot and no supersession
620            // contributes only to its thread's layer entry, so it's left
621            // undecoded in the thread's cold bucket (most of a
622            // thread-heavy file). Its row says it isn't a fact.
623            if let Some(r) = e.v9 {
624                if r.kind == ROW_KIND_TURN
625                    && r.thread_hash != 0
626                    && r.key_hash == 0
627                    && r.flags & ROW_SUPERSEDED == 0
628                {
629                    layers
630                        .cold_turns
631                        .entry(r.thread_hash)
632                        .or_default()
633                        .push(e.id);
634                    non_facts.push(e.id);
635                    continue;
636                }
637            }
638            let m = self.read_memory_meta(e)?;
639            layers.add(&m);
640            if !is_ann_kind(&m) {
641                non_facts.push(e.id);
642            }
643            if m.metadata.get(MNEMO_META_SUPERSEDED_BY).is_some() {
644                superseded.insert(e.id);
645                continue;
646            }
647            if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
648                let slot = Self::key_slot_for(&m, k);
649                // Catalog order is insertion order; a later live record
650                // with the same slot would only exist if a prior build
651                // of this code allowed it. Keep the later id (newer
652                // ULID), in case of a tie.
653                keys.entry(slot)
654                    .and_modify(|cur| {
655                        if *cur < e.id {
656                            *cur = e.id;
657                        }
658                    })
659                    .or_insert(e.id);
660            }
661        }
662        self.key_index = Some(keys);
663        self.superseded = Some(superseded);
664        self.layers = Some(layers);
665        if let Some(ann) = &mut self.ann {
666            let mut pruned = 0usize;
667            for id in non_facts {
668                if ann.contains(id) {
669                    ann.remove(id);
670                    pruned += 1;
671                }
672            }
673            if pruned > 0 {
674                self.dirty_index = true;
675                self.index_persist.changes += pruned;
676            }
677        }
678        Ok(())
679    }
680}
681
682/// RB-001.24: the error for caller metadata that sets an engine key.
683pub(super) fn reserved_key_error(key: &str) -> MnemoError {
684    MnemoError::Invalid(format!(
685        "metadata key \"{key}\" is reserved: keys starting with \"{}\" are set by the engine",
686        crate::memory::MNEMO_META_NS
687    ))
688}