mnemo_engine/store/crud.rs
1//! Public CRUD (remember, get, delete, embedder), and the handle caches (key slots, superseded set, thread layers).
2
3use super::*;
4
5/// R-004.2: where the thread layers and the onboarding briefing live,
6/// built by the handle's one lazy scan (`ensure_key_caches`, together with
7/// the 12.1 key caches) and then kept current by `put` and `delete`. It
8/// lets `about`, `essence`, `ingest_thread`, `thread_export` and
9/// thread-pinned recall read just their own records instead of decrypting
10/// the whole file each time.
11#[derive(Debug, Default)]
12pub(super) struct LayerCache {
13 /// Live essence item ids.
14 pub(super) essence: std::collections::HashSet<u128>,
15 /// Live ids tagged `metadata.area = "onboarding"` (superseded versions
16 /// included; `about` drops those when it reads them).
17 pub(super) onboarding: std::collections::HashSet<u128>,
18 /// Per `_mnemo.thread`: its records and archive state.
19 pub(super) threads: std::collections::HashMap<String, ThreadCache>,
20 /// Thread → (the thread it continues, the link record's id).
21 pub(super) links: std::collections::HashMap<String, (String, u128)>,
22 /// FMT-9e: on v9, thread-turn records the lazy scan didn't decode,
23 /// by their row's `thread_hash`. A thread's bucket is decoded into
24 /// `threads` the first time that thread is used
25 /// ([`Mnemo::warm_thread`]), so a cold handle never decodes (or
26 /// decompresses) the turns of threads it doesn't touch.
27 pub(super) cold_turns: std::collections::HashMap<u32, Vec<u128>>,
28}
29
30/// One thread's records (turns, essence items, link), the turn ids already
31/// archived (with how many records each has: long turns are chunked), and
32/// the highest `_mnemo.seq` seen.
33#[derive(Debug, Default)]
34pub(super) struct ThreadCache {
35 pub(super) records: std::collections::HashSet<u128>,
36 pub(super) turns: std::collections::HashMap<String, u32>,
37 pub(super) max_seq: Option<u64>,
38}
39
40pub(super) fn meta_str<'a>(m: &'a Memory, key: &str) -> Option<&'a str> {
41 m.metadata.get(key).and_then(|v| v.as_str())
42}
43
44pub(super) fn is_onboarding(m: &Memory) -> bool {
45 meta_str(m, "area").is_some_and(|s| s.eq_ignore_ascii_case("onboarding"))
46}
47
48impl LayerCache {
49 pub(super) fn add(&mut self, m: &Memory) {
50 let id = m.id.0;
51 let kind = meta_str(m, MNEMO_META_KIND);
52 if kind == Some(MNEMO_META_KIND_ESSENCE) {
53 self.essence.insert(id);
54 }
55 if is_onboarding(m) {
56 self.onboarding.insert(id);
57 }
58 let Some(thread) = meta_str(m, MNEMO_META_THREAD) else {
59 return;
60 };
61 let tc = self.threads.entry(thread.to_string()).or_default();
62 tc.records.insert(id);
63 if kind == Some(MNEMO_META_KIND_TURN) {
64 if let Some(turn_id) = meta_str(m, MNEMO_META_TURN_ID) {
65 *tc.turns.entry(turn_id.to_string()).or_insert(0) += 1;
66 }
67 if let Some(seq) = m.metadata.get(MNEMO_META_SEQ).and_then(|v| v.as_u64()) {
68 tc.max_seq = Some(tc.max_seq.map_or(seq, |s| s.max(seq)));
69 }
70 }
71 if kind == Some(MNEMO_META_KIND_THREAD_LINK) {
72 if let Some(from) = meta_str(m, MNEMO_META_CONTINUES) {
73 self.links
74 .entry(thread.to_string())
75 .or_insert_with(|| (from.to_string(), id));
76 }
77 }
78 }
79
80 /// Undo [`LayerCache::add`] for the stored version of a record that is
81 /// being overwritten or deleted. A thread with no records left is
82 /// dropped, so a forgotten thread's `_mnemo.seq` restarts at 0. Partial
83 /// deletes keep the old maximum (sequence gaps are harmless).
84 pub(super) fn remove(&mut self, m: &Memory) {
85 let id = m.id.0;
86 self.essence.remove(&id);
87 self.onboarding.remove(&id);
88 let Some(thread) = meta_str(m, MNEMO_META_THREAD) else {
89 return;
90 };
91 if self.links.get(thread).is_some_and(|(_, link)| *link == id) {
92 self.links.remove(thread);
93 }
94 let Some(tc) = self.threads.get_mut(thread) else {
95 return;
96 };
97 tc.records.remove(&id);
98 if meta_str(m, MNEMO_META_KIND) == Some(MNEMO_META_KIND_TURN) {
99 if let Some(turn_id) = meta_str(m, MNEMO_META_TURN_ID) {
100 if let Some(n) = tc.turns.get_mut(turn_id) {
101 *n -= 1;
102 if *n == 0 {
103 tc.turns.remove(turn_id);
104 }
105 }
106 }
107 }
108 if tc.records.is_empty() {
109 self.threads.remove(thread);
110 }
111 }
112}
113
114impl Mnemo {
115 /// Register a text embedder for this handle. Enables
116 /// [`Mnemo::remember_text`] and the text-side `RecallRequest`
117 /// convenience path. Does two checks before storing the embedder:
118 ///
119 /// 1. **Dimension check.** The embedder's `identity().dimensions`
120 /// must equal [`Mnemo::dimensions`] (the width this database
121 /// was created with). Mismatch returns
122 /// [`MnemoError::DimensionMismatch`].
123 /// 2. **Identity check (EMBED-ID-1).** Every file records the
124 /// identity (name, dimensions, normalize) of the first embedder
125 /// attached to it, in a reserved, encrypted record that isn't a
126 /// memory: `len`, `count`, `stats`, recall, `about` and `get`
127 /// never see it, and compaction, the rekeyed exports and the
128 /// passphrase import carry it. After that, any other identity,
129 /// even at the same width (another model, or another revision of
130 /// the same one), returns [`MnemoError::EmbedderMismatch`]: a
131 /// store is bound to the model that first wrote to it, so a new
132 /// model needs a new store.
133 ///
134 /// **Files with a manifest** (`mnemo init`, `seed.json`) also carry
135 /// the identity on the manifest at `metadata.embedder = { name,
136 /// dimensions, normalize }` (the legacy flat `embedder_name` /
137 /// `embedder_dimensions` / `embedder_normalize` keys are read as an
138 /// alias). The first registration upserts it there too, as before.
139 /// If the manifest and the record disagree, this returns
140 /// [`MnemoError::Invalid`] naming both.
141 ///
142 /// **Files from before EMBED-ID-1** have no record: the first
143 /// embedder attached after the upgrade is recorded. The check
144 /// protects from then on; it can't tell which model made the older
145 /// vectors.
146 ///
147 /// **Read-only handles.** `set_embedder` on an `open_read_only`
148 /// handle performs both checks and adopts the embedder in memory;
149 /// it never writes. Readers (CLI `--text`, agent-side `recall_text`)
150 /// can therefore use the text path next to a writer that holds
151 /// the exclusive lock.
152 ///
153 /// On every failure path the handle's embedder slot is left
154 /// unchanged. On success the embedder is wrapped in an `Arc<dyn
155 /// Embedder>` and consulted by every text-side call made through
156 /// this handle. The slot is per-handle and not persisted — a
157 /// fresh open must re-register (same identity, so the check
158 /// round-trips cleanly).
159 pub fn set_embedder<E: Embedder + 'static>(&mut self, embedder: E) -> Result<()> {
160 let identity = embedder.identity();
161 if identity.dimensions != self.dimensions {
162 return Err(MnemoError::DimensionMismatch {
163 expected: self.dimensions,
164 got: identity.dimensions,
165 });
166 }
167
168 let (onboarding, on_manifest, upsert_target, on_record) = self.embedder_records()?;
169
170 if let (Some(manifest), Some(record)) = (&on_manifest, &on_record) {
171 if manifest != record {
172 return Err(embed::identity_conflict_err(manifest, record));
173 }
174 }
175 if let Some(stored) = on_record.as_ref().or(on_manifest.as_ref()) {
176 if *stored != identity {
177 return Err(embed::identity_mismatch_err(stored, &identity));
178 }
179 }
180
181 // Record what's missing. A read-only handle writes nothing, but
182 // still adopts the embedder in memory.
183 if !self.read_only {
184 let mut wrote = false;
185 if on_manifest.is_none() {
186 if let Some(idx) = upsert_target {
187 let mut m = onboarding[idx].clone();
188 embed::write_identity_into_metadata(&mut m.metadata, &identity);
189 // Reuse the memory's existing id so this is an in-place
190 // update of the scaffold manifest, not a new entry.
191 self.put(m)?;
192 wrote = true;
193 }
194 }
195 if on_record.is_none() {
196 self.record_identity(&identity)?;
197 wrote = true;
198 }
199 if wrote {
200 self.flush()?;
201 }
202 }
203
204 self.embedder = Some(std::sync::Arc::new(embedder));
205 Ok(())
206 }
207
208 /// Where the file records an embedder identity (EMBED-ID-1): the
209 /// onboarding memories, the manifest's identity if it has one, the first
210 /// onboarding memory without one (the upsert target), and the reserved
211 /// record's identity.
212 #[allow(clippy::type_complexity)]
213 fn embedder_records(
214 &mut self,
215 ) -> Result<(
216 Vec<Memory>,
217 Option<embed::EmbedderIdentity>,
218 Option<usize>,
219 Option<embed::EmbedderIdentity>,
220 )> {
221 let onboarding = self.about_in(crate::memory::VisibilitySet::ALL)?;
222 let mut on_manifest: Option<embed::EmbedderIdentity> = None;
223 let mut upsert_target: Option<usize> = None;
224 for (i, m) in onboarding.iter().enumerate() {
225 if let Some(saw) = embed::read_identity_from_metadata(&m.metadata) {
226 on_manifest = Some(saw);
227 break;
228 }
229 if upsert_target.is_none() {
230 upsert_target = Some(i);
231 }
232 }
233 let on_record = self.recorded_identity()?;
234 Ok((onboarding, on_manifest, upsert_target, on_record))
235 }
236
237 /// Whether [`Mnemo::set_embedder`] with this identity would write to the
238 /// file (to record the identity where it's missing). A shared handle
239 /// attaches an already recorded embedder in a read window (RB-007.6).
240 pub(super) fn embedder_would_record(
241 &mut self,
242 identity: &embed::EmbedderIdentity,
243 ) -> Result<bool> {
244 if identity.dimensions != self.dimensions {
245 // `set_embedder` refuses it without writing.
246 return Ok(false);
247 }
248 let (_, on_manifest, upsert_target, on_record) = self.embedder_records()?;
249 Ok((on_manifest.is_none() && upsert_target.is_some()) || on_record.is_none())
250 }
251
252 /// The embedder identity this file records (EMBED-ID-1), or `None`
253 /// before the first `set_embedder` on a read-write handle. Reads the
254 /// reserved record without touching any counter.
255 pub fn embedder_identity(&mut self) -> Result<Option<embed::EmbedderIdentity>> {
256 self.recorded_identity()
257 }
258
259 /// Borrow the currently-registered embedder, if any. Returns
260 /// `None` when no embedder has been set via
261 /// [`Mnemo::set_embedder`]. Bindings use this to drive their own
262 /// text-side convenience methods without duplicating the identity
263 /// check.
264 pub fn embedder(&self) -> Option<&dyn Embedder> {
265 self.embedder.as_deref()
266 }
267
268 /// Shared handle to the registered embedder, cloned so the caller
269 /// can drop the borrow on this `Mnemo` before invoking `embed()`.
270 /// FIX-1 / PY-1 (R-001.11): the Python binding needs this so an
271 /// embedder callback that re-enters `Mnemo` (e.g. a Python
272 /// embedder that calls `db.dimensions()` or `db.cache_get(...)`)
273 /// doesn't hit PyO3's "Already borrowed" — we clone the Arc under
274 /// a short `&self` borrow, release that borrow, then invoke
275 /// `embed()` with no `&mut Mnemo` held.
276 pub fn embedder_arc(&self) -> Option<std::sync::Arc<dyn Embedder>> {
277 self.embedder.clone()
278 }
279
280 /// Embed `content` with the registered embedder and `remember` the
281 /// result. Returns the new memory's ULID. Convenience path for
282 /// callers who have a string, want it stored, and don't need any
283 /// of the per-memory options [`Memory`] exposes (agent, importance,
284 /// metadata, ttl, scope) — build a [`Memory`] by hand and call
285 /// [`Mnemo::remember`] for those.
286 ///
287 /// Fails with [`MnemoError::Invalid`] if no embedder has been
288 /// registered on this handle (call [`Mnemo::set_embedder`] first);
289 /// with [`MnemoError::DimensionMismatch`] if the embedder returned
290 /// the wrong width for this database.
291 pub fn remember_text(&mut self, content: &str, memory_type: MemoryType) -> Result<Ulid> {
292 if self.read_only {
293 return Err(MnemoError::ReadOnly);
294 }
295 let embedder = self.embedder.as_ref().ok_or_else(|| {
296 MnemoError::Invalid(
297 "no embedder registered on this handle; call set_embedder first".into(),
298 )
299 })?;
300 let mut vectors = embedder.embed(&[content])?;
301 let vector = vectors.pop().ok_or_else(|| {
302 MnemoError::Invalid("embedder returned no vector for the input text".into())
303 })?;
304 if vector.len() != self.dimensions {
305 return Err(MnemoError::DimensionMismatch {
306 expected: self.dimensions,
307 got: vector.len(),
308 });
309 }
310 let memory = Memory::new(content.to_string(), memory_type, vector);
311 self.remember(memory)
312 }
313
314 /// Store a memory. Returns its ULID. If the memory's ID is nil a fresh
315 /// one is assigned; an existing ID overwrites in place.
316 ///
317 /// Metadata keys starting with `_mnemo.` are the engine's (supersession,
318 /// keys, threads, kinds): a new memory carrying one is refused with
319 /// [`MnemoError::Invalid`]. An in-place overwrite may carry the stored
320 /// record's own `_mnemo.*` keys through unchanged (the usual `get`,
321 /// change a field, `remember`), but can't add, change or drop one.
322 pub fn remember(&mut self, memory: Memory) -> Result<Ulid> {
323 if self.read_only {
324 return Err(MnemoError::ReadOnly);
325 }
326 self.check_caller_metadata(&memory)?;
327 self.put(memory)
328 }
329
330 /// RB-001.24: the caller's metadata may not set engine state. For a
331 /// memory with a nil or unknown id, any `_mnemo.*` key is refused; for
332 /// an overwrite in place, the `_mnemo.*` keys must equal the stored
333 /// record's.
334 pub(super) fn check_caller_metadata(&mut self, memory: &Memory) -> Result<()> {
335 // 13a: visibility and freshness are the caller's to set (checked);
336 // every other `_mnemo.*` key is the engine's.
337 for (k, v) in memory.metadata.iter().filter(|(k, _)| is_caller_key(k)) {
338 crate::memory::check_caller_key(k, v).map_err(MnemoError::Invalid)?;
339 }
340 let reserved: Vec<(&String, &Value)> = memory
341 .metadata
342 .iter()
343 .filter(|(k, _)| k.starts_with(crate::memory::MNEMO_META_NS) && !is_caller_key(k))
344 .collect();
345 if reserved.is_empty() {
346 return Ok(());
347 }
348 let stored = if memory.id.is_nil() {
349 None
350 } else {
351 match self.index.get(memory.id.0, &self.catalog) {
352 Some(idx) if !self.catalog[idx].deleted => {
353 let entry = self.catalog[idx].clone();
354 Some(self.read_memory(&entry)?)
355 }
356 _ => None,
357 }
358 };
359 if let Some(old) = stored {
360 let old_reserved: Vec<(&String, &Value)> = old
361 .metadata
362 .iter()
363 .filter(|(k, _)| k.starts_with(crate::memory::MNEMO_META_NS) && !is_caller_key(k))
364 .collect();
365 if old_reserved == reserved {
366 return Ok(());
367 }
368 }
369 Err(reserved_key_error(reserved[0].0))
370 }
371
372 /// Fetch a memory by ID.
373 pub fn get(&mut self, id: &Ulid) -> Result<Memory> {
374 let idx = self
375 .index
376 .get(id.0, &self.catalog)
377 .ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
378 let entry = self.catalog[idx].clone();
379 if entry.deleted {
380 return Err(MnemoError::NotFound(id.to_string()));
381 }
382 self.read_memory(&entry)
383 }
384
385 /// Soft-delete a memory (tombstoned; space reclaimed by `compact`).
386 pub fn delete(&mut self, id: &Ulid) -> Result<()> {
387 if self.read_only {
388 return Err(MnemoError::ReadOnly);
389 }
390 let idx = self
391 .index
392 .get(id.0, &self.catalog)
393 .ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
394 // R-004.2 / 12.1: with the handle caches built, read the record
395 // being deleted (one decrypt) so the key slot, superseded set and
396 // thread layers drop exactly what it contributed.
397 let entry = self.catalog[idx].clone();
398 if self.layers.is_some() && !entry.deleted {
399 let old = self.read_memory(&entry)?;
400 self.note_removed(&old)?;
401 }
402 if !self.catalog[idx].deleted {
403 self.catalog[idx].deleted = true;
404 self.dirty_catalog = true;
405 self.touch(idx);
406 // FMT-9f: drop its document from the stats in place.
407 if let (Some(stats), Some(r)) = (self.bm25.stats.as_mut(), entry.v9) {
408 if r.doc != 0 {
409 stats.remove(r.doc);
410 }
411 }
412 if self.is_v9() {
413 self.kill_record(idx, &entry);
414 self.kill_vector(idx, &entry);
415 } else {
416 self.release_pages(entry.start_page, entry.page_count as u64);
417 }
418 }
419 if let Some(ann) = &mut self.ann {
420 if ann.contains(id.0) {
421 ann.remove(id.0);
422 self.dirty_index = true;
423 self.index_persist.changes += 1;
424 }
425 }
426 // FIX-1 R-001.8 part 2: we can't tell whether this delete
427 // removed the last memory of its agent without a decrypt
428 // (the catalog entry doesn't carry `agent_id` in format v8),
429 // so just invalidate the lazy set. The next `stats()` call
430 // rebuilds from live memories.
431 self.agent_set = None;
432 // Phase 12.1: delete of a keyed record removes it from the slot,
433 // and delete of a superseded record frees a `history` entry. When
434 // the caches are built, `note_removed` above did that exactly;
435 // otherwise they're built from scratch on first use anyway.
436 Ok(())
437 }
438
439 /// Keep the handle caches (12.1 key slots, superseded set, R-004.2
440 /// layers) current after `put` writes `m`. No-op until they're built.
441 /// FMT-9e: a cold thread is warmed first, so its layer entry is whole.
442 pub(super) fn note_added(&mut self, m: &Memory) -> Result<()> {
443 if let Some(thread) = meta_str(m, MNEMO_META_THREAD) {
444 self.warm_thread(thread)?;
445 }
446 let id = m.id.0;
447 if let (Some(keys), Some(sup)) = (self.key_index.as_mut(), self.superseded.as_mut()) {
448 if m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) {
449 sup.insert(id);
450 } else if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
451 keys.insert(Self::key_slot_for(m, k), id);
452 }
453 }
454 if let Some(layers) = self.layers.as_mut() {
455 layers.add(m);
456 }
457 Ok(())
458 }
459
460 /// Undo [`Mnemo::note_added`] for the stored version of a record that's
461 /// about to be overwritten or deleted.
462 pub(super) fn note_removed(&mut self, m: &Memory) -> Result<()> {
463 if let Some(thread) = meta_str(m, MNEMO_META_THREAD) {
464 self.warm_thread(thread)?;
465 }
466 let id = m.id.0;
467 if let (Some(keys), Some(sup)) = (self.key_index.as_mut(), self.superseded.as_mut()) {
468 sup.remove(&id);
469 if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
470 let slot = Self::key_slot_for(m, k);
471 if keys.get(&slot) == Some(&id) {
472 keys.remove(&slot);
473 }
474 }
475 }
476 if let Some(layers) = self.layers.as_mut() {
477 layers.remove(m);
478 }
479 Ok(())
480 }
481
482 /// FMT-9e: decode `thread`'s cold turn records (see
483 /// [`LayerCache::cold_turns`]) into the layer cache. Every record in
484 /// the thread's hash bucket is decoded and filed under its own thread,
485 /// so a hash collision just warms both threads. A no-op when the
486 /// caches aren't built or the bucket is already warm.
487 pub(super) fn warm_thread(&mut self, thread: &str) -> Result<()> {
488 self.warm_thread_decoded(thread).map(|_| ())
489 }
490
491 /// [`Mnemo::warm_thread`], returning the records it decoded (without
492 /// vectors), so a caller about to read the thread decodes each once.
493 pub(super) fn warm_thread_decoded(
494 &mut self,
495 thread: &str,
496 ) -> Result<std::collections::HashMap<u128, Memory>> {
497 let mut decoded = std::collections::HashMap::new();
498 let h = fnv1a32(&[thread.as_bytes()]);
499 let Some(ids) = self.layers.as_mut().and_then(|l| l.cold_turns.remove(&h)) else {
500 return Ok(decoded);
501 };
502 for e in self.live_entries(ids) {
503 let m = self.read_memory_meta(&e)?;
504 if let Some(layers) = self.layers.as_mut() {
505 layers.add(&m);
506 }
507 decoded.insert(e.id, m);
508 }
509 Ok(decoded)
510 }
511
512 /// Decode a record's memory without its vector or live access stats:
513 /// what the handle caches need (metadata, scope, agent), without the
514 /// v9 vector-column read [`Mnemo::read_memory`] adds.
515 pub(super) fn read_memory_meta(&mut self, e: &CatalogEntry) -> Result<Memory> {
516 let bytes = self.read_record(e)?;
517 self.decode_memory(&bytes)
518 }
519
520 /// The live catalog entries for `ids` (ids that are gone are skipped).
521 pub(super) fn live_entries(&self, ids: impl IntoIterator<Item = u128>) -> Vec<CatalogEntry> {
522 ids.into_iter()
523 .filter_map(|id| {
524 self.index
525 .get(id, &self.catalog)
526 .map(|i| self.catalog[i].clone())
527 })
528 .filter(|e| !e.deleted)
529 .collect()
530 }
531
532 /// Catalog entries of the essence items, from the handle caches.
533 pub(super) fn essence_entries(&mut self) -> Result<Vec<CatalogEntry>> {
534 self.ensure_key_caches()?;
535 let ids: Vec<u128> = self
536 .layers
537 .as_ref()
538 .map(|l| l.essence.iter().copied().collect())
539 .unwrap_or_default();
540 Ok(self.live_entries(ids))
541 }
542
543 /// Catalog entries of every record filed under `thread`, from the
544 /// handle caches.
545 pub(super) fn thread_entries(&mut self, thread: &str) -> Result<Vec<CatalogEntry>> {
546 Ok(self.thread_entries_decoded(thread)?.0)
547 }
548
549 /// [`Mnemo::thread_entries`], plus the records that warming a cold
550 /// thread just decoded (see [`Mnemo::warm_thread_decoded`]).
551 pub(super) fn thread_entries_decoded(
552 &mut self,
553 thread: &str,
554 ) -> Result<(Vec<CatalogEntry>, std::collections::HashMap<u128, Memory>)> {
555 self.ensure_key_caches()?;
556 let decoded = self.warm_thread_decoded(thread)?;
557 let ids: Vec<u128> = self
558 .layers
559 .as_ref()
560 .and_then(|l| l.threads.get(thread))
561 .map(|tc| tc.records.iter().copied().collect())
562 .unwrap_or_default();
563 Ok((self.live_entries(ids), decoded))
564 }
565
566 // --- Phase 12.1: edit in place + keyed supersession -----------------
567
568 /// Scope-aware key-slot identifier for `memory` and a caller key.
569 ///
570 /// * [`Scope::Private`] → owned by the memory's `agent_id`.
571 /// * [`Scope::Shared`] → one slot across the whole file.
572 pub(super) fn key_slot_for(memory: &Memory, key: &str) -> crate::store::KeySlot {
573 let scope_owner = match memory.scope {
574 Scope::Private => Some(memory.agent_id.clone()),
575 Scope::Shared => None,
576 };
577 crate::store::KeySlot {
578 scope_owner,
579 key: key.to_string(),
580 }
581 }
582
583 /// The handle's one lazy scan: builds the 12.1 `key_index` +
584 /// `superseded` caches and the R-004.2 layer cache (essence ids,
585 /// onboarding ids, threads, links) from live records, and prunes
586 /// non-fact ids from a loaded ANN index (files written before v0.10.0
587 /// indexed every kind). Decrypts every live record once per handle;
588 /// `put` and `delete` keep the result current after that.
589 pub(super) fn ensure_key_caches(&mut self) -> Result<()> {
590 if self.key_index.is_some() && self.superseded.is_some() && self.layers.is_some() {
591 return Ok(());
592 }
593 let entries: Vec<CatalogEntry> = self
594 .catalog
595 .iter()
596 .filter(|e| !e.deleted)
597 .cloned()
598 .collect();
599 let mut keys: std::collections::HashMap<crate::store::KeySlot, u128> =
600 std::collections::HashMap::new();
601 let mut superseded: std::collections::HashSet<u128> = std::collections::HashSet::new();
602 let mut layers = LayerCache::default();
603 let mut non_facts: Vec<u128> = Vec::new();
604 for e in &entries {
605 // FMT-9e: a v9 turn row with no key slot and no supersession
606 // contributes only to its thread's layer entry, so it's left
607 // undecoded in the thread's cold bucket (most of a
608 // thread-heavy file). Its row says it isn't a fact.
609 if let Some(r) = e.v9 {
610 if r.kind == ROW_KIND_TURN
611 && r.thread_hash != 0
612 && r.key_hash == 0
613 && r.flags & ROW_SUPERSEDED == 0
614 {
615 layers
616 .cold_turns
617 .entry(r.thread_hash)
618 .or_default()
619 .push(e.id);
620 non_facts.push(e.id);
621 continue;
622 }
623 }
624 let m = self.read_memory_meta(e)?;
625 layers.add(&m);
626 if !is_ann_kind(&m) {
627 non_facts.push(e.id);
628 }
629 if m.metadata.get(MNEMO_META_SUPERSEDED_BY).is_some() {
630 superseded.insert(e.id);
631 continue;
632 }
633 if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
634 let slot = Self::key_slot_for(&m, k);
635 // Catalog order is insertion order; a later live record
636 // with the same slot would only exist if a prior build
637 // of this code allowed it. Keep the later id (newer
638 // ULID), in case of a tie.
639 keys.entry(slot)
640 .and_modify(|cur| {
641 if *cur < e.id {
642 *cur = e.id;
643 }
644 })
645 .or_insert(e.id);
646 }
647 }
648 self.key_index = Some(keys);
649 self.superseded = Some(superseded);
650 self.layers = Some(layers);
651 if let Some(ann) = &mut self.ann {
652 let mut pruned = 0usize;
653 for id in non_facts {
654 if ann.contains(id) {
655 ann.remove(id);
656 pruned += 1;
657 }
658 }
659 if pruned > 0 {
660 self.dirty_index = true;
661 self.index_persist.changes += pruned;
662 }
663 }
664 Ok(())
665 }
666}
667
668/// RB-001.24: the error for caller metadata that sets an engine key.
669pub(super) fn reserved_key_error(key: &str) -> MnemoError {
670 MnemoError::Invalid(format!(
671 "metadata key \"{key}\" is reserved: keys starting with \"{}\" are set by the engine",
672 crate::memory::MNEMO_META_NS
673 ))
674}