mnemo_engine/result_cache.rs
1//! Exact-key result cache (Phase 10.1 of the level-up plan).
2//!
3//! **This is NOT the page cache** (that's `mnemo/src/cache.rs`, which
4//! caches decrypted pages inside the pager). This module implements a
5//! *user-facing key/value cache* backed by the same encrypted-page
6//! storage engine as the memory catalog — a `.mnemo` file becomes both a
7//! durable agent memory store AND a durable exact-key cache for LLM
8//! tool-call results, prompt→completion pairs, HTTP responses, and
9//! anything else an agent wants to memoize.
10//!
11//! ## Model
12//!
13//! Each cache entry lives at a `(namespace, key)` pair. The engine
14//! SHA-256-hashes `key` — callers pass the raw input string (a prompt,
15//! a tool-call JSON, a URL, whatever), the engine handles the hashing.
16//! Namespaces isolate independent caches inside one file so `llm` and
17//! `http` and `tool` never collide.
18//!
19//! On-disk layout mirrors the memory catalog + record split:
20//!
21//! - **Cache directory** — a serialized `Vec<CacheDirectoryEntry>` in
22//! its own encrypted page run, pointed at by
23//! [`crate::format::Header`]'s `cache_start` / `cache_pages` /
24//! `cache_len` fields (added in format v8). Small per-entry
25//! overhead; the directory is what a lookup scans.
26//! - **Cache records** — the payloads. One [`CacheEntry`] per record,
27//! serialized as MessagePack and stored in the same encrypted-page
28//! append-only region the memory records use. Never rewritten on
29//! read; access stats (`accessed_at`, `access_count`) live on the
30//! directory entry only (the v5 recall-side-trick).
31//!
32//! ## Flush policy
33//!
34//! [`CacheFlushPolicy`] is per-handle:
35//!
36//! - `Strict` (default): every `cache_put` dirties the directory; the
37//! next call to `Mnemo::flush()` persists everything in one
38//! WAL-committed transaction.
39//! - `Batched { max_dirty, max_age }`: `cache_put` still dirties the
40//! directory in memory, but the engine tracks pending-dirty count
41//! and first-dirty timestamp. When either threshold trips, the
42//! engine internally calls `flush()` — persisting the whole
43//! transaction (including any pending memory writes, which is safe
44//! because they were queued for a flush anyway; the only effect is
45//! earlier-than-user-expected durability).
46//!
47//! **Memory writes are never batched** — `remember` behaves exactly as
48//! it always did, requiring a user-driven `flush()` to persist. Only
49//! cache mutations get the relaxed lane.
50//!
51//! ## Crash semantics
52//!
53//! Losing unflushed (Batched-lane) cache entries on crash is a cache
54//! *miss*, never corruption. Every commit — whether Strict or Batched —
55//! goes through the same single-fsync WAL transaction that already
56//! backs memory writes. The append-only-snapshot invariant is untouched.
57
58use serde::{Deserialize, Serialize};
59use sha2::{Digest, Sha256};
60use std::collections::HashMap;
61use std::time::{Duration, Instant};
62
63// --- Public tunables and types -------------------------------------------
64
65/// Default per-namespace entry-count budget. Once a namespace holds this
66/// many live entries, `cache_put` evicts the LRU entry before inserting.
67pub const DEFAULT_MAX_ENTRIES: usize = 10_000;
68
69/// Default per-namespace byte budget (values + directory overhead).
70/// Once exceeded, `cache_put` evicts LRU until the total falls back
71/// under the cap.
72pub const DEFAULT_MAX_BYTES: u64 = 64 * 1024 * 1024;
73
74/// Default `max_dirty` for [`CacheFlushPolicy::Batched`].
75pub const DEFAULT_BATCH_MAX_DIRTY: usize = 128;
76
77/// Default `max_age` for [`CacheFlushPolicy::Batched`].
78pub const DEFAULT_BATCH_MAX_AGE: Duration = Duration::from_secs(5);
79
80/// Default similarity threshold for [`crate::Mnemo::cache_get_semantic`].
81/// Lookups below this are treated as misses. Chosen conservatively to
82/// avoid false hits on subtly-different prompts; callers can override
83/// per-lookup. Model-dependent — text-embedding-ada-002 and BGE at
84/// this threshold rarely surface unrelated content, but if you're
85/// working with a lossier embedder consider raising it.
86pub const DEFAULT_SEMANTIC_THRESHOLD: f32 = 0.97;
87
88/// Length in chars of the human-readable key preview stored on every
89/// directory entry. Long enough to be recognizable, short enough that
90/// the directory scan stays fast on large caches.
91pub const KEY_PREVIEW_MAX_CHARS: usize = 120;
92
93/// Options accepted by `Mnemo::cache_put`. Defaults describe an
94/// untyped, never-expiring, non-cost-hinted entry.
95#[derive(Clone, Debug)]
96pub struct CachePutOpts {
97 /// Free-form content-type label. Convention: `"json"`, `"text"`,
98 /// `"bytes"`, or a MIME string. Not interpreted by the engine —
99 /// preserved verbatim on `cache_get` so callers can round-trip
100 /// their own type discipline.
101 pub content_type: String,
102 /// Time-to-live in seconds from the moment of `cache_put`. `None`
103 /// means "never expires by itself" (still evictable via LRU or
104 /// namespace purge).
105 pub ttl_secs: Option<u64>,
106 /// Optional hint of how expensive the miss-side computation was,
107 /// in milliseconds. Recorded per-entry; today used only as a
108 /// tie-breaker for future cost-aware eviction (see the GDSF
109 /// follow-up note in the level-up plan).
110 pub cost_hint_ms: Option<u32>,
111}
112
113impl Default for CachePutOpts {
114 fn default() -> Self {
115 Self {
116 content_type: "text".into(),
117 ttl_secs: None,
118 cost_hint_ms: None,
119 }
120 }
121}
122
123/// Options for [`crate::Mnemo::cache_put_semantic`]. Same as
124/// [`CachePutOpts`] plus a **required** `model` string — a hit from a
125/// different embedding model's cache is a semantics bug, not a win, so
126/// the engine forces callers to declare the model and matches it on
127/// lookup.
128///
129/// The vector itself is a separate argument to `cache_put_semantic`
130/// (not inside opts) so callers who already own the vector don't have
131/// to `.clone()` it into the struct.
132#[derive(Clone, Debug)]
133pub struct SemanticCachePutOpts {
134 /// Free-form content-type label recorded on the entry.
135 pub content_type: String,
136 /// TTL in seconds. Omit for no expiry.
137 pub ttl_secs: Option<u64>,
138 /// Cost hint recorded on the entry (see [`CachePutOpts::cost_hint_ms`]).
139 pub cost_hint_ms: Option<u32>,
140 /// Embedding model identifier. Matched exactly on lookup; a
141 /// different model's cache is invisible to this query. Convention:
142 /// `"text-embedding-3-small"`, `"bge-large-en-v1.5"`, etc.
143 pub model: String,
144}
145
146impl SemanticCachePutOpts {
147 /// Construct with just the required `model`; all other fields
148 /// default (content_type = "text", no TTL, no cost hint).
149 pub fn new(model: impl Into<String>) -> Self {
150 Self {
151 content_type: "text".into(),
152 ttl_secs: None,
153 cost_hint_ms: None,
154 model: model.into(),
155 }
156 }
157}
158
159/// A returned cache hit. `accessed_at` / `access_count` are updated on
160/// the directory *before* this is constructed, so the values here
161/// reflect the post-touch state (mirroring the recall-time convention).
162#[derive(Clone, Debug)]
163pub struct CachedValue {
164 /// Value bytes as stored by `cache_put`.
165 pub value: Vec<u8>,
166 /// Content-type label as stored by `cache_put`.
167 pub content_type: String,
168 /// Unix seconds when this entry was created.
169 pub created_at: i64,
170 /// Unix seconds when this entry was last accessed (this hit).
171 pub accessed_at: i64,
172 /// Number of hits recorded on this entry (including the current one).
173 pub access_count: u32,
174 /// TTL configured on `cache_put`, if any.
175 pub ttl_secs: Option<u64>,
176}
177
178/// Per-namespace eviction budget. `cache_put` enforces both caps: on
179/// insert, the LRU is evicted until *both* live-entry count and live
180/// byte total fit under the caps.
181#[derive(Clone, Copy, Debug)]
182pub struct CacheBudget {
183 /// Hard cap on live directory entries for this namespace.
184 pub max_entries: usize,
185 /// Hard cap on live payload bytes for this namespace.
186 pub max_bytes: u64,
187}
188
189impl Default for CacheBudget {
190 fn default() -> Self {
191 Self {
192 max_entries: DEFAULT_MAX_ENTRIES,
193 max_bytes: DEFAULT_MAX_BYTES,
194 }
195 }
196}
197
198/// Per-handle flush policy for cache writes. Memory writes ignore this
199/// and always behave as `Strict`.
200#[derive(Clone, Copy, Debug, Default)]
201pub enum CacheFlushPolicy {
202 /// Every `cache_put` dirties the directory; the caller's next
203 /// `Mnemo::flush()` persists everything.
204 #[default]
205 Strict,
206 /// The engine internally calls `flush()` when either the pending
207 /// dirty count reaches `max_dirty` OR the first pending dirty
208 /// entry is older than `max_age`.
209 Batched {
210 /// Auto-flush after this many pending cache mutations.
211 max_dirty: usize,
212 /// Auto-flush this long after the first pending mutation.
213 max_age: Duration,
214 },
215}
216
217/// Summary statistics for a namespace (or the whole cache).
218#[derive(Clone, Copy, Debug, Default)]
219pub struct CacheStats {
220 /// Live (non-deleted, non-expired) entries.
221 pub entries: usize,
222 /// Live payload bytes.
223 pub bytes: u64,
224 /// Hits recorded on this handle since open.
225 pub hits: u64,
226 /// Misses recorded on this handle since open (includes TTL-expired
227 /// and non-existent keys — both look the same to the caller).
228 pub misses: u64,
229 /// Hits divided by (hits + misses); `0.0` if no lookups.
230 pub hit_rate: f64,
231 /// Directory entries evicted (by LRU-under-budget) since open.
232 pub evictions: u64,
233}
234
235// --- On-disk types (crate-internal) --------------------------------------
236
237/// The MessagePack record body of a cache entry. Stored in the same
238/// encrypted-page region as memory records.
239///
240/// `vector` and `model` are the semantic-cache extension (Phase 10.2).
241/// `#[serde(default)]` keeps pre-semantic v8 entries — where the
242/// on-disk record has no vector / no model — round-trip-safe under
243/// the extended schema: rmp-serde decodes missing tail fields as
244/// their `Default::default()`, so old entries surface as `None`/`None`
245/// (exactly the "exact-key only, no vector" state we want).
246#[derive(Serialize, Deserialize, Clone, Debug)]
247pub(crate) struct CacheEntry {
248 pub namespace: String,
249 pub key_hash: [u8; 32],
250 pub key_preview: String,
251 pub value: Vec<u8>,
252 pub content_type: String,
253 pub created_at: i64,
254 pub accessed_at: i64,
255 pub access_count: u32,
256 pub ttl_secs: Option<u64>,
257 pub cost_hint_ms: Option<u32>,
258 pub size_bytes: u64,
259 /// Semantic-cache: embedding vector for this entry, or `None` for
260 /// pure exact-key entries.
261 #[serde(default)]
262 pub vector: Option<Vec<f32>>,
263 /// Semantic-cache: embedding model identifier this vector was
264 /// computed under, or `None` for pure exact-key entries. Matched
265 /// exactly on `cache_get_semantic` — different-model hits are bugs.
266 #[serde(default)]
267 pub model: Option<String>,
268}
269
270/// Directory entry — the "catalog row" for a cache record. Access stats
271/// live here (not on the [`CacheEntry`]) so a hit doesn't require a full
272/// record rewrite (mirrors the v5 recall trick).
273///
274/// `size_bytes` is duplicated from the entry body so budget enforcement
275/// runs against the directory alone without decrypting record pages.
276#[derive(Serialize, Deserialize, Clone, Debug)]
277pub(crate) struct CacheDirectoryEntry {
278 pub namespace: String,
279 pub key_hash: [u8; 32],
280 pub start_page: u64,
281 pub page_count: u32,
282 pub len: u32,
283 pub deleted: bool,
284 pub accessed_at: i64,
285 pub access_count: u32,
286 pub size_bytes: u64,
287 pub ttl_secs: Option<u64>,
288 pub created_at: i64,
289 /// Semantic-cache: embedding vector duplicated onto the directory
290 /// entry so lookup can scan without reading (and decrypting) every
291 /// record body. `None` on pure exact-key entries. `#[serde(default)]`
292 /// keeps pre-semantic v8 directories decoding under the extended
293 /// schema.
294 #[serde(default)]
295 pub vector: Option<Vec<f32>>,
296 /// Semantic-cache: model identifier. `None` on pure exact-key entries.
297 #[serde(default)]
298 pub model: Option<String>,
299 /// FMT-9: generation the entry's record pages were sealed under (v9
300 /// page AAD). Always `0`, and not serialized, on v8 files, so the v8
301 /// directory encoding is byte-for-byte unchanged.
302 #[serde(default, skip_serializing_if = "crate::format::is_zero_u64")]
303 pub gen: u64,
304}
305
306impl CacheDirectoryEntry {
307 /// Has the entry passed its TTL at `now`? An entry with no TTL is
308 /// never expired.
309 pub(crate) fn is_expired(&self, now: i64) -> bool {
310 match self.ttl_secs {
311 Some(ttl) => now.saturating_sub(self.created_at) as u64 >= ttl,
312 None => false,
313 }
314 }
315}
316
317// --- Key hashing ---------------------------------------------------------
318
319/// SHA-256 hash a caller-supplied cache key. Deterministic and platform-
320/// independent (endianness-safe because the input is a byte string, not
321/// a typed value).
322pub fn hash_key(key: &str) -> [u8; 32] {
323 let mut h = Sha256::new();
324 h.update(key.as_bytes());
325 h.finalize().into()
326}
327
328/// Truncate a key to at most [`KEY_PREVIEW_MAX_CHARS`] chars (not
329/// bytes), suffixing `…` when cut. Preserves char-boundary safety for
330/// UTF-8 previews.
331pub fn key_preview(key: &str) -> String {
332 if key.chars().count() <= KEY_PREVIEW_MAX_CHARS {
333 return key.to_string();
334 }
335 let mut out: String = key.chars().take(KEY_PREVIEW_MAX_CHARS - 1).collect();
336 out.push('…');
337 out
338}
339
340// --- In-memory directory index -------------------------------------------
341
342/// In-memory index over the on-disk directory: given `(namespace,
343/// key_hash)`, find the position of the live entry in the directory
344/// `Vec`. Absent entries and deleted tombstones are not indexed.
345#[derive(Debug, Default)]
346pub(crate) struct DirectoryIndex {
347 // Namespace name → key_hash → index into the directory Vec.
348 by_ns: HashMap<String, HashMap<[u8; 32], usize>>,
349}
350
351impl DirectoryIndex {
352 pub(crate) fn new() -> Self {
353 Self {
354 by_ns: HashMap::new(),
355 }
356 }
357
358 /// Build an index over a directory `Vec`, skipping deleted entries.
359 pub(crate) fn build(entries: &[CacheDirectoryEntry]) -> Self {
360 let mut idx = Self::new();
361 for (i, e) in entries.iter().enumerate() {
362 if !e.deleted {
363 idx.insert(&e.namespace, e.key_hash, i);
364 }
365 }
366 idx
367 }
368
369 pub(crate) fn insert(&mut self, namespace: &str, key_hash: [u8; 32], pos: usize) {
370 self.by_ns
371 .entry(namespace.to_string())
372 .or_default()
373 .insert(key_hash, pos);
374 }
375
376 pub(crate) fn remove(&mut self, namespace: &str, key_hash: &[u8; 32]) {
377 if let Some(ns) = self.by_ns.get_mut(namespace) {
378 ns.remove(key_hash);
379 }
380 }
381
382 pub(crate) fn get(&self, namespace: &str, key_hash: &[u8; 32]) -> Option<usize> {
383 self.by_ns.get(namespace)?.get(key_hash).copied()
384 }
385}
386
387// --- In-memory hit/miss counters -----------------------------------------
388
389/// Per-handle hit/miss/eviction counters. Persisted portion is aggregate
390/// evictions (via the directory tombstones themselves); hits and misses
391/// are in-memory only and reset on reopen.
392#[derive(Debug, Default)]
393pub(crate) struct CounterState {
394 pub hits: u64,
395 pub misses: u64,
396 pub evictions: u64,
397}
398
399// --- Batched-lane bookkeeping --------------------------------------------
400
401/// Tracks pending-batch state for [`CacheFlushPolicy::Batched`]. In
402/// `Strict` mode all fields stay zero; the store consults this before
403/// deciding whether a `cache_put` should trigger an auto-flush.
404#[derive(Debug, Default)]
405pub(crate) struct BatchState {
406 /// Count of cache mutations since the last commit.
407 pub dirty_count: usize,
408 /// Wallclock of the first mutation since the last commit — used to
409 /// evaluate `max_age`.
410 pub first_dirty_at: Option<Instant>,
411}
412
413impl BatchState {
414 pub(crate) fn record_dirty(&mut self) {
415 self.dirty_count += 1;
416 if self.first_dirty_at.is_none() {
417 self.first_dirty_at = Some(Instant::now());
418 }
419 }
420
421 pub(crate) fn reset(&mut self) {
422 self.dirty_count = 0;
423 self.first_dirty_at = None;
424 }
425
426 /// Should the store auto-flush now, given the current batch state
427 /// and policy? Always `false` for `Strict`.
428 pub(crate) fn should_auto_flush(&self, policy: &CacheFlushPolicy) -> bool {
429 match policy {
430 CacheFlushPolicy::Strict => false,
431 CacheFlushPolicy::Batched { max_dirty, max_age } => {
432 if self.dirty_count >= *max_dirty {
433 return true;
434 }
435 if let Some(t0) = self.first_dirty_at {
436 if t0.elapsed() >= *max_age {
437 return true;
438 }
439 }
440 false
441 }
442 }
443 }
444}
445
446// --- Unit tests ---------------------------------------------------------
447
448#[cfg(test)]
449mod tests {
450 use super::*;
451
452 #[test]
453 fn hash_key_is_deterministic_and_32_bytes() {
454 let a = hash_key("hello");
455 let b = hash_key("hello");
456 assert_eq!(a, b);
457 assert_eq!(a.len(), 32);
458 // Different input → different hash (SHA-256 collision would be a
459 // major cryptographic event; this just guards typo bugs).
460 assert_ne!(a, hash_key("hello!"));
461 }
462
463 #[test]
464 fn key_preview_truncates_with_ellipsis() {
465 let short = "hi there";
466 assert_eq!(key_preview(short), short);
467 let long = "x".repeat(200);
468 let out = key_preview(&long);
469 assert_eq!(out.chars().count(), KEY_PREVIEW_MAX_CHARS);
470 assert!(out.ends_with('…'));
471 }
472
473 #[test]
474 fn directory_index_locates_live_entries() {
475 let entries = vec![
476 CacheDirectoryEntry {
477 namespace: "llm".into(),
478 key_hash: [1u8; 32],
479 start_page: 100,
480 page_count: 1,
481 len: 42,
482 deleted: false,
483 accessed_at: 0,
484 access_count: 0,
485 size_bytes: 42,
486 ttl_secs: None,
487 created_at: 0,
488 vector: None,
489 model: None,
490 gen: 0,
491 },
492 CacheDirectoryEntry {
493 namespace: "http".into(),
494 key_hash: [2u8; 32],
495 start_page: 101,
496 page_count: 1,
497 len: 42,
498 deleted: true, // tombstone — should be skipped
499 accessed_at: 0,
500 access_count: 0,
501 size_bytes: 42,
502 ttl_secs: None,
503 created_at: 0,
504 vector: None,
505 model: None,
506 gen: 0,
507 },
508 ];
509 let idx = DirectoryIndex::build(&entries);
510 assert_eq!(idx.get("llm", &[1u8; 32]), Some(0));
511 assert_eq!(idx.get("http", &[2u8; 32]), None, "tombstones not indexed");
512 }
513
514 #[test]
515 fn batch_state_triggers_on_count() {
516 let mut b = BatchState::default();
517 let p = CacheFlushPolicy::Batched {
518 max_dirty: 3,
519 max_age: Duration::from_secs(3600),
520 };
521 b.record_dirty();
522 b.record_dirty();
523 assert!(!b.should_auto_flush(&p));
524 b.record_dirty();
525 assert!(b.should_auto_flush(&p));
526 }
527
528 #[test]
529 fn ttl_expiry_matches_created_at() {
530 let now = 1_000_000_i64;
531 let fresh = CacheDirectoryEntry {
532 namespace: "".into(),
533 key_hash: [0u8; 32],
534 start_page: 0,
535 page_count: 0,
536 len: 0,
537 deleted: false,
538 accessed_at: 0,
539 access_count: 0,
540 size_bytes: 0,
541 ttl_secs: Some(60),
542 created_at: now - 30,
543 vector: None,
544 model: None,
545 gen: 0,
546 };
547 assert!(!fresh.is_expired(now), "30s old with 60s TTL is fresh");
548 let stale = CacheDirectoryEntry {
549 created_at: now - 120,
550 ..fresh.clone()
551 };
552 assert!(stale.is_expired(now), "120s old with 60s TTL is expired");
553 let no_ttl = CacheDirectoryEntry {
554 ttl_secs: None,
555 ..fresh
556 };
557 assert!(!no_ttl.is_expired(now + 1_000_000_000));
558 }
559}