mnemo_engine/lib.rs
1//! # Mnemo
2//!
3//! Mnemo is an **encrypted, single-file, portable agent-memory engine**.
4//!
5//! A whole memory store — vectors, content, metadata, and the multi-signal
6//! recall machinery an agent needs — lives in one file you can copy, back up,
7//! or hand to another process. The file is encrypted at rest with a two-tier
8//! key hierarchy: an Argon2id key-encryption key (KEK) derived from a
9//! passphrase wraps a random data-encryption key (DEK), and the DEK encrypts
10//! every page with AES-256-GCM.
11//!
12//! ## Quick start
13//!
14//! ```no_run
15//! use mnemo_engine::{Mnemo, MnemoConfig, Memory, MemoryType, RecallRequest};
16//!
17//! # fn main() -> mnemo_engine::Result<()> {
18//! let cfg = MnemoConfig { dimensions: 3, ..Default::default() };
19//! let mut db = Mnemo::create("agent.mnemo", "correct horse battery", cfg)?;
20//!
21//! db.remember(
22//! Memory::new("the user prefers dark mode", MemoryType::Semantic, vec![0.1, 0.2, 0.9])
23//! .with_agent("assistant-1")
24//! .with_importance(0.8),
25//! )?;
26//! db.flush()?;
27//!
28//! let hits = db.recall(&RecallRequest::new(vec![0.1, 0.2, 0.9]).top_k(5))?;
29//! for h in hits {
30//! println!("{:.3} {}", h.score, h.memory.content);
31//! }
32//! # Ok(())
33//! # }
34//! ```
35//!
36//! ## What is and is not built
37//!
38//! This crate implements a real, tested core: the encrypted single-file
39//! storage engine, the crypto layer, the agent-memory model, multi-signal
40//! recall, an IVF+PQ approximate-nearest-neighbour index, a write-ahead log,
41//! snapshot-based point-in-time recovery, a bounded LRU page cache, and the
42//! [`Session`] conversation wrapper. Exact brute-force search remains
43//! available as the ground-truth baseline; a built index makes
44//! [`Mnemo::recall`] sub-linear. Each [`Mnemo::flush`] is one atomic,
45//! WAL-committed transaction and a restorable snapshot — [`Mnemo::restore_to`]
46//! rewinds the database to any past transaction. Python and TypeScript
47//! language bindings are the one documented roadmap item — see the README.
48
49#![forbid(unsafe_code)]
50#![warn(missing_docs)]
51
52mod cache;
53mod crypto;
54mod embed;
55mod embed_cmd;
56mod error;
57mod format;
58mod freelist;
59mod index;
60mod keyslot;
61pub mod mcp;
62mod memory;
63mod pager;
64mod result_cache;
65mod session;
66mod store;
67mod text;
68mod transcript;
69mod veccol;
70mod wal;
71
72pub use crypto::KdfParams;
73pub use embed::{Embedder, EmbedderIdentity};
74pub use embed_cmd::{EmbedCmd, DEFAULT_EMBED_TIMEOUT};
75pub use error::{MnemoError, Result};
76pub use index::{IndexConfig, IndexInfo};
77pub use memory::{
78 status as essence_status, CommitBatch, CommitReport, EssenceOp, EssenceOpKind, EssenceScope,
79 EssenceSection, ExportedTurn, Filter, FilterTerm, IngestReport, KeyScope, KeyedMemory, Memory,
80 MemoryPatch, MemoryType, Metric, Scope, ScoreWeights, ThreadTurn, TurnPolicy, WriteOutcome,
81 CONTINUATION_TIMELINE_ENTRIES, ESSENCE_DEFAULT_RENDER_BYTES, ESSENCE_SECTION_MAX_BYTES,
82 ESSENCE_TIMELINE_MAX_BYTES, ESSENCE_TRANSITIONS, FINDING_TRANSITIONS, MNEMO_META_CONTINUES,
83 MNEMO_META_INVALID_AT, MNEMO_META_KEY, MNEMO_META_KIND, MNEMO_META_KIND_ESSENCE,
84 MNEMO_META_KIND_THREAD_LINK, MNEMO_META_KIND_TURN, MNEMO_META_LENGTH, MNEMO_META_NS,
85 MNEMO_META_PART, MNEMO_META_REFS, MNEMO_META_ROLE, MNEMO_META_SECTION, MNEMO_META_SEQ,
86 MNEMO_META_SHA256, MNEMO_META_STATUS, MNEMO_META_STATUS_HISTORY, MNEMO_META_SUPERSEDED_BY,
87 MNEMO_META_SUPERSEDES, MNEMO_META_THREAD, MNEMO_META_TURN_ID, MNEMO_META_UPDATED_AT,
88 TASK_TRANSITIONS, THREAD_TOOL_RESULT_TRUNCATE_BYTES, THREAD_TURN_CHUNK_BYTES,
89};
90pub use result_cache::{
91 CacheBudget, CacheFlushPolicy, CachePutOpts, CacheStats, CachedValue, SemanticCachePutOpts,
92 DEFAULT_BATCH_MAX_AGE, DEFAULT_BATCH_MAX_DIRTY, DEFAULT_MAX_BYTES, DEFAULT_MAX_ENTRIES,
93 DEFAULT_SEMANTIC_THRESHOLD,
94};
95pub use session::{Role, Session, Turn};
96#[doc(hidden)]
97pub use store::BudgetReport;
98pub use store::{
99 CompactReport, Mnemo, MnemoConfig, RecallRequest, RecallResponse, RecallResult, SnapshotInfo,
100 Stats, DEFAULT_KEYWORD_WEIGHT, DEFAULT_TURN_COMPRESS_AFTER_SECS, DEFAULT_VECTOR_CACHE_BUDGET,
101 INDEX_PERSIST_EVERY_CHANGES, INDEX_PERSIST_EVERY_COMMITS, V9_ANN_MIN_FACTS, V9_RERANK_PER_HIT,
102};
103pub use transcript::{read_transcript, TranscriptFormat};
104
105/// Re-export of [`ulid::Ulid`], the identifier type used for memories.
106pub use ulid::Ulid;
107
108// --- Fsync counter -----------------------------------------------------
109//
110// Instrumentation plumbing for the Phase 6 top-3 cost #2 fix: the
111// planner wants to know how many `sync_all` calls each `flush()` issues
112// before deciding whether to batch any of them. Every fsync call site
113// bumps a thread-local counter; `Mnemo::flush` snapshots it before and
114// after, exposing the delta on `Stats::fsyncs_last_flush`. Thread-local
115// so handles on separate threads don't poison each other's counts; the
116// engine's one-writer-at-a-time locking means a single handle sees a
117// coherent stream.
118
119mod fsync_counter {
120 use std::cell::Cell;
121
122 thread_local! {
123 static COUNT: Cell<u64> = const { Cell::new(0) };
124 }
125
126 /// Call once per `sync_all`.
127 pub(crate) fn bump() {
128 COUNT.with(|c| c.set(c.get().wrapping_add(1)));
129 }
130
131 /// Monotonic counter snapshot for this thread.
132 pub(crate) fn count() -> u64 {
133 COUNT.with(|c| c.get())
134 }
135}
136
137pub(crate) use fsync_counter::{bump as bump_fsync, count as fsync_count};
138
139/// Build-time cryptographic backend information.
140///
141/// Populated at compile time from the active `cfg` flags. On ARMv8
142/// builds compiled with `--cfg aes_armv8 --cfg polyval_armv8` (set by
143/// the repo's `.cargo/config.toml` and both CI workflows) the `aes`
144/// and `polyval` crates emit hardware AES + PMULL instructions; without
145/// those flags they fall back to a constant-time software path that
146/// runs at roughly 5× the cost per page.
147///
148/// Published through [`build_info`], exposed by `mnemo info` on the
149/// CLI and `mnemo.build_info()` from the Python binding. The point is
150/// verification — nobody should have to profile a release build to find
151/// out whether it's running software AES.
152#[derive(Debug, Clone, Copy, PartialEq, Eq)]
153pub struct BuildInfo {
154 /// Package version (`CARGO_PKG_VERSION`), e.g. `"0.5.1"`.
155 pub version: &'static str,
156 /// Target triple this binary was built for, e.g.
157 /// `"aarch64-apple-darwin"`.
158 pub target: &'static str,
159 /// Human-readable AES backend. One of:
160 /// - `"armv8 hardware"` — ARM with `cfg(aes_armv8)`
161 /// - `"armv8 software"` — ARM without the cfg; constant-time fallback
162 /// - `"x86_64 AES-NI (runtime detect)"` — x86_64 with runtime
163 /// AES-NI detection in the `aes` crate
164 /// - `"portable software"` — any other target
165 pub aes_backend: &'static str,
166 /// Human-readable PMULL/CLMUL backend used by GHASH (part of AES-GCM).
167 /// Same shape as `aes_backend` but for `polyval`.
168 pub polyval_backend: &'static str,
169}
170
171/// Report this build's cryptographic backend configuration.
172///
173/// Cheap — all fields are `&'static str` populated at compile time.
174/// Safe to call from a hot path if, for some reason, that's useful.
175pub fn build_info() -> BuildInfo {
176 BuildInfo {
177 version: env!("CARGO_PKG_VERSION"),
178 target: TARGET_TRIPLE,
179 aes_backend: aes_backend_name(),
180 polyval_backend: polyval_backend_name(),
181 }
182}
183
184/// Target triple baked in via the `TARGET` env var that `cargo` sets
185/// for the host compile. Falls back to `"unknown"` if the var isn't
186/// set at build time (shouldn't happen under cargo, but a build.rs-free
187/// implementation keeps this crate build-script-free, so we use a
188/// cfg-based dispatch as the second-best).
189const TARGET_TRIPLE: &str = {
190 // `env!("TARGET")` isn't set by cargo for crates; it's set by rustc
191 // only for `std` builds. We approximate from `target_arch` /
192 // `target_os` cfgs so this stays build-script-free. Good enough
193 // for an info line; precise triples are a `cargo` or build.rs job.
194 if cfg!(all(target_arch = "aarch64", target_os = "macos")) {
195 "aarch64-apple-darwin"
196 } else if cfg!(all(target_arch = "aarch64", target_os = "linux")) {
197 "aarch64-unknown-linux-gnu"
198 } else if cfg!(all(target_arch = "x86_64", target_os = "macos")) {
199 "x86_64-apple-darwin"
200 } else if cfg!(all(target_arch = "x86_64", target_os = "linux")) {
201 "x86_64-unknown-linux-gnu"
202 } else if cfg!(all(target_arch = "x86_64", target_os = "windows")) {
203 "x86_64-pc-windows-msvc"
204 } else {
205 "unknown"
206 }
207};
208
209const fn aes_backend_name() -> &'static str {
210 if cfg!(aes_armv8) {
211 "armv8 hardware"
212 } else if cfg!(target_arch = "aarch64") {
213 // Software fallback is the dangerous case this whole mechanism
214 // exists to surface — flag it clearly so `mnemo info` makes it
215 // easy to notice a build picked up the slow path by mistake.
216 "armv8 software (constant-time fallback)"
217 } else if cfg!(target_arch = "x86_64") {
218 // `aes` 0.8.4 on x86_64 picks AES-NI vs the software path at
219 // runtime via the `cpufeatures` crate, so we can't tell from
220 // cfg alone which path a given invocation takes. In practice
221 // every x86_64 CPU shipped since ~2010 has AES-NI.
222 "x86_64 AES-NI (runtime detect)"
223 } else {
224 "portable software"
225 }
226}
227
228const fn polyval_backend_name() -> &'static str {
229 if cfg!(polyval_armv8) {
230 "armv8 PMULL hardware"
231 } else if cfg!(target_arch = "aarch64") {
232 "armv8 software (constant-time fallback)"
233 } else if cfg!(target_arch = "x86_64") {
234 "x86_64 CLMUL (runtime detect)"
235 } else {
236 "portable software"
237 }
238}
239
240/// Test-only crash-injection surface used by the Phase 1.3 proptest
241/// (`mnemo/tests/proptest_store.rs`). Compiled in under `cfg(test)`
242/// and the `failpoints` feature only; absent from release builds.
243///
244/// **Not part of the stable public API.** The double-underscore
245/// convention makes the "internal" contract obvious at the call site.
246#[cfg(any(test, feature = "failpoints"))]
247#[doc(hidden)]
248pub mod __failpoints {
249 /// Arm the pager and WAL commit path to fail on the Nth
250 /// subsequent write. `n = 0` fails the very next write; `n = -1`
251 /// (or any negative) disables. One-shot — disarms on trigger.
252 ///
253 /// Every write is checked before the byte hits the file, so a
254 /// triggered failure is guaranteed torn-BEFORE, never
255 /// torn-AFTER — critical for the "either pre-flush or post-flush"
256 /// invariant the property test asserts.
257 pub fn set_writes_until_fail(n: i64) {
258 crate::pager::failpoints::set_writes_until_fail(n)
259 }
260}
261
262/// WAL entry points for the PERF-1 allocation test
263/// (`tests/wal_recover_alloc.rs`), which needs its own global allocator and
264/// so can't live in this `forbid(unsafe_code)` crate. Not a stable API.
265#[doc(hidden)]
266pub mod __wal {
267 /// Page size of the WAL region.
268 pub const PAGE_SIZE: usize = crate::format::PAGE_SIZE;
269
270 /// One WAL frame: a home page number and its page image.
271 pub type Frame = (u64, Vec<u8>);
272
273 /// Bytes a transaction of `n` page frames occupies.
274 pub fn txn_byte_len(n: usize) -> u64 {
275 crate::wal::txn_byte_len(n, PAGE_SIZE)
276 }
277
278 /// Commit `frames` as transaction `txn_id` at the start of a region.
279 pub fn commit(
280 file: &mut std::fs::File,
281 wal_pages: u64,
282 txn_id: u64,
283 frames: &[Frame],
284 ) -> crate::Result<()> {
285 crate::wal::commit(file, 0, wal_pages, txn_id, frames)
286 }
287
288 /// Run open-time recovery over a region starting at page 0; returns the
289 /// frames to replay, if any.
290 pub fn recover(
291 file: &mut std::fs::File,
292 wal_pages: u64,
293 wal_seq: u64,
294 ) -> crate::Result<Option<Vec<Frame>>> {
295 crate::wal::recover(file, 0, wal_pages, wal_seq)
296 }
297}
298
299/// Internal parse surfaces exposed for cargo-fuzz targets (Phase 1.2).
300///
301/// **Not part of the stable public API.** Each function feeds an
302/// arbitrary byte slice into a parse/decode path that a hostile
303/// `.mnemo` file (or a hostile MCP client) can send us, and returns
304/// `Result<()>` so a fuzz target only asserts "no panic." Marked
305/// `#[doc(hidden)]` so `cargo doc` doesn't advertise it; do not import
306/// from application code.
307#[doc(hidden)]
308pub mod __fuzz {
309 /// Parse a `.mnemo` header page (page 0, plaintext) from arbitrary
310 /// bytes. This is the pre-passphrase attack surface — a hostile
311 /// file hits it before any DEK exists.
312 pub fn parse_header(bytes: &[u8]) -> crate::Result<()> {
313 crate::format::Header::from_page(bytes).map(|_| ())
314 }
315
316 /// Replay the WAL scan over an arbitrary byte region. Delegates
317 /// to [`crate::wal::recover_bytes`]; a corrupt / truncated /
318 /// adversarial region must return `Err` or `Ok(None/Some(_))`,
319 /// never panic.
320 pub fn wal_recover_bytes(region: &[u8], wal_seq: u64) -> crate::Result<()> {
321 crate::wal::recover_bytes(region, wal_seq).map(|_| ())
322 }
323
324 /// Decode a `Memory` record body from arbitrary bytes.
325 pub fn decode_memory(bytes: &[u8]) -> crate::Result<()> {
326 rmp_serde::from_slice::<crate::Memory>(bytes)
327 .map(|_| ())
328 .map_err(|e| crate::MnemoError::Serialize(e.to_string()))
329 }
330
331 /// FMT-9e: decode a v9 `Memory` record body from arbitrary bytes: a
332 /// zstd frame (a compressed thread turn) is decompressed first, under
333 /// the same content-size bound the store applies.
334 pub fn decode_v9_memory(bytes: &[u8]) -> crate::Result<()> {
335 let plain = crate::store::plain_record(bytes)?;
336 rmp_serde::from_slice::<crate::Memory>(&plain)
337 .map(|_| ())
338 .map_err(|e| crate::MnemoError::Serialize(e.to_string()))
339 }
340
341 /// Decode a `CacheEntry` record body from arbitrary bytes. Type is
342 /// `pub(crate)`, so this wrapper hides it behind a unit return.
343 pub fn decode_cache_entry(bytes: &[u8]) -> crate::Result<()> {
344 rmp_serde::from_slice::<crate::result_cache::CacheEntry>(bytes)
345 .map(|_| ())
346 .map_err(|e| crate::MnemoError::Serialize(e.to_string()))
347 }
348
349 /// Decode a cache-directory page-run body (a `Vec<CacheDirectoryEntry>`)
350 /// from arbitrary bytes.
351 pub fn decode_cache_directory(bytes: &[u8]) -> crate::Result<()> {
352 rmp_serde::from_slice::<Vec<crate::result_cache::CacheDirectoryEntry>>(bytes)
353 .map(|_| ())
354 .map_err(|e| crate::MnemoError::Serialize(e.to_string()))
355 }
356
357 /// Parse a single MCP stdio JSON-RPC request line. Malformed
358 /// input returns `Err(String)`; the fuzz target only asserts
359 /// no panic and that the server would not die on this line.
360 pub fn parse_mcp_request_line(line: &str) -> Result<(), String> {
361 crate::mcp::__fuzz_parse_request_line(line)
362 }
363}