Skip to main content

mnemo_engine/
lib.rs

1//! # Mnemo
2//!
3//! Mnemo is an **encrypted, single-file, portable agent-memory engine**.
4//!
5//! A whole memory store — vectors, content, metadata, and the multi-signal
6//! recall machinery an agent needs — lives in one file you can copy, back up,
7//! or hand to another process. The file is encrypted at rest with a two-tier
8//! key hierarchy: an Argon2id key-encryption key (KEK) derived from a
9//! passphrase wraps a random data-encryption key (DEK), and the DEK encrypts
10//! every page with AES-256-GCM.
11//!
12//! ## Quick start
13//!
14//! ```no_run
15//! use mnemo_engine::{Mnemo, MnemoConfig, Memory, MemoryType, RecallRequest};
16//!
17//! # fn main() -> mnemo_engine::Result<()> {
18//! let cfg = MnemoConfig { dimensions: 3, ..Default::default() };
19//! let mut db = Mnemo::create("agent.mnemo", "correct horse battery", cfg)?;
20//!
21//! db.remember(
22//!     Memory::new("the user prefers dark mode", MemoryType::Semantic, vec![0.1, 0.2, 0.9])
23//!         .with_agent("assistant-1")
24//!         .with_importance(0.8),
25//! )?;
26//! db.flush()?;
27//!
28//! let hits = db.recall(&RecallRequest::new(vec![0.1, 0.2, 0.9]).top_k(5))?;
29//! for h in hits {
30//!     println!("{:.3}  {}", h.score, h.memory.content);
31//! }
32//! # Ok(())
33//! # }
34//! ```
35//!
36//! ## What is and is not built
37//!
38//! This crate implements a real, tested core: the encrypted single-file
39//! storage engine, the crypto layer, the agent-memory model, multi-signal
40//! recall, an IVF+PQ approximate-nearest-neighbour index, a write-ahead log,
41//! snapshot-based point-in-time recovery, a bounded LRU page cache, and the
42//! [`Session`] conversation wrapper. Exact brute-force search remains
43//! available as the ground-truth baseline; a built index makes
44//! [`Mnemo::recall`] sub-linear. Each [`Mnemo::flush`] is one atomic,
45//! WAL-committed transaction and a restorable snapshot — [`Mnemo::restore_to`]
46//! rewinds the database to any past transaction. Python and TypeScript
47//! language bindings are the one documented roadmap item — see the README.
48
49#![forbid(unsafe_code)]
50#![warn(missing_docs)]
51
52mod cache;
53mod crypto;
54mod embed;
55mod embed_cmd;
56mod error;
57mod format;
58mod freelist;
59mod index;
60mod keyslot;
61pub mod mcp;
62mod memory;
63mod pager;
64mod result_cache;
65mod session;
66mod store;
67mod text;
68mod transcript;
69mod veccol;
70mod wal;
71
72pub use crypto::KdfParams;
73pub use embed::{Embedder, EmbedderIdentity};
74pub use embed_cmd::{EmbedCmd, DEFAULT_EMBED_TIMEOUT};
75pub use error::{MnemoError, Result};
76pub use index::{IndexConfig, IndexInfo};
77pub use memory::{
78    status as essence_status, CommitBatch, CommitReport, EssenceOp, EssenceOpKind, EssenceScope,
79    EssenceSection, ExportedTurn, Filter, FilterTerm, IngestReport, KeyScope, KeyedMemory, Memory,
80    MemoryPatch, MemoryType, Metric, Scope, ScoreWeights, ThreadTurn, TurnPolicy, WriteOutcome,
81    CONTINUATION_TIMELINE_ENTRIES, ESSENCE_DEFAULT_RENDER_BYTES, ESSENCE_SECTION_MAX_BYTES,
82    ESSENCE_TIMELINE_MAX_BYTES, ESSENCE_TRANSITIONS, FINDING_TRANSITIONS, MNEMO_META_CONTINUES,
83    MNEMO_META_INVALID_AT, MNEMO_META_KEY, MNEMO_META_KIND, MNEMO_META_KIND_ESSENCE,
84    MNEMO_META_KIND_THREAD_LINK, MNEMO_META_KIND_TURN, MNEMO_META_LENGTH, MNEMO_META_NS,
85    MNEMO_META_PART, MNEMO_META_REFS, MNEMO_META_ROLE, MNEMO_META_SECTION, MNEMO_META_SEQ,
86    MNEMO_META_SHA256, MNEMO_META_STATUS, MNEMO_META_STATUS_HISTORY, MNEMO_META_SUPERSEDED_BY,
87    MNEMO_META_SUPERSEDES, MNEMO_META_THREAD, MNEMO_META_TURN_ID, MNEMO_META_UPDATED_AT,
88    TASK_TRANSITIONS, THREAD_TOOL_RESULT_TRUNCATE_BYTES, THREAD_TURN_CHUNK_BYTES,
89};
90pub use result_cache::{
91    CacheBudget, CacheFlushPolicy, CachePutOpts, CacheStats, CachedValue, SemanticCachePutOpts,
92    DEFAULT_BATCH_MAX_AGE, DEFAULT_BATCH_MAX_DIRTY, DEFAULT_MAX_BYTES, DEFAULT_MAX_ENTRIES,
93    DEFAULT_SEMANTIC_THRESHOLD,
94};
95pub use session::{Role, Session, Turn};
96#[doc(hidden)]
97pub use store::BudgetReport;
98pub use store::{
99    CloseError, CompactReport, Mnemo, MnemoConfig, RecallMode, RecallRequest, RecallResponse,
100    RecallResult, SnapshotInfo, Stats, StatsSummary, DEFAULT_KEYWORD_WEIGHT,
101    DEFAULT_TURN_COMPRESS_AFTER_SECS, DEFAULT_VECTOR_CACHE_BUDGET, INDEX_PERSIST_EVERY_CHANGES,
102    INDEX_PERSIST_EVERY_COMMITS, V9_ANN_MIN_FACTS, V9_RERANK_PER_HIT,
103};
104pub use transcript::{read_transcript, TranscriptFormat};
105
106/// Re-export of [`ulid::Ulid`], the identifier type used for memories.
107pub use ulid::Ulid;
108
109// --- Fsync counter -----------------------------------------------------
110//
111// Instrumentation plumbing for the Phase 6 top-3 cost #2 fix: the
112// planner wants to know how many `sync_all` calls each `flush()` issues
113// before deciding whether to batch any of them. Every fsync call site
114// bumps a thread-local counter; `Mnemo::flush` snapshots it before and
115// after, exposing the delta on `Stats::fsyncs_last_flush`. Thread-local
116// so handles on separate threads don't poison each other's counts; the
117// engine's one-writer-at-a-time locking means a single handle sees a
118// coherent stream.
119
120mod fsync_counter {
121    use std::cell::Cell;
122
123    thread_local! {
124        static COUNT: Cell<u64> = const { Cell::new(0) };
125    }
126
127    /// Call once per `sync_all`.
128    pub(crate) fn bump() {
129        COUNT.with(|c| c.set(c.get().wrapping_add(1)));
130    }
131
132    /// Monotonic counter snapshot for this thread.
133    pub(crate) fn count() -> u64 {
134        COUNT.with(|c| c.get())
135    }
136}
137
138pub(crate) use fsync_counter::{bump as bump_fsync, count as fsync_count};
139
140/// Build-time cryptographic backend information.
141///
142/// Populated at compile time from the active `cfg` flags. On ARMv8
143/// builds compiled with `--cfg aes_armv8 --cfg polyval_armv8` (set by
144/// the repo's `.cargo/config.toml` and both CI workflows) the `aes`
145/// and `polyval` crates emit hardware AES + PMULL instructions; without
146/// those flags they fall back to a constant-time software path that
147/// runs at roughly 5× the cost per page.
148///
149/// Published through [`build_info`], exposed by `mnemo info` on the
150/// CLI and `mnemo.build_info()` from the Python binding. The point is
151/// verification — nobody should have to profile a release build to find
152/// out whether it's running software AES.
153#[derive(Debug, Clone, Copy, PartialEq, Eq)]
154pub struct BuildInfo {
155    /// Package version (`CARGO_PKG_VERSION`), e.g. `"0.5.1"`.
156    pub version: &'static str,
157    /// Target triple this binary was built for, e.g.
158    /// `"aarch64-apple-darwin"`.
159    pub target: &'static str,
160    /// Human-readable AES backend. One of:
161    /// - `"armv8 hardware"` — ARM with `cfg(aes_armv8)`
162    /// - `"armv8 software"` — ARM without the cfg; constant-time fallback
163    /// - `"x86_64 AES-NI (runtime detect)"` — x86_64 with runtime
164    ///   AES-NI detection in the `aes` crate
165    /// - `"portable software"` — any other target
166    pub aes_backend: &'static str,
167    /// Human-readable PMULL/CLMUL backend used by GHASH (part of AES-GCM).
168    /// Same shape as `aes_backend` but for `polyval`.
169    pub polyval_backend: &'static str,
170}
171
172/// Report this build's cryptographic backend configuration.
173///
174/// Cheap — all fields are `&'static str` populated at compile time.
175/// Safe to call from a hot path if, for some reason, that's useful.
176pub fn build_info() -> BuildInfo {
177    BuildInfo {
178        version: env!("CARGO_PKG_VERSION"),
179        target: TARGET_TRIPLE,
180        aes_backend: aes_backend_name(),
181        polyval_backend: polyval_backend_name(),
182    }
183}
184
185/// Target triple baked in via the `TARGET` env var that `cargo` sets
186/// for the host compile. Falls back to `"unknown"` if the var isn't
187/// set at build time (shouldn't happen under cargo, but a build.rs-free
188/// implementation keeps this crate build-script-free, so we use a
189/// cfg-based dispatch as the second-best).
190const TARGET_TRIPLE: &str = {
191    // `env!("TARGET")` isn't set by cargo for crates; it's set by rustc
192    // only for `std` builds. We approximate from `target_arch` /
193    // `target_os` cfgs so this stays build-script-free. Good enough
194    // for an info line; precise triples are a `cargo` or build.rs job.
195    if cfg!(all(target_arch = "aarch64", target_os = "macos")) {
196        "aarch64-apple-darwin"
197    } else if cfg!(all(target_arch = "aarch64", target_os = "linux")) {
198        "aarch64-unknown-linux-gnu"
199    } else if cfg!(all(target_arch = "x86_64", target_os = "macos")) {
200        "x86_64-apple-darwin"
201    } else if cfg!(all(target_arch = "x86_64", target_os = "linux")) {
202        "x86_64-unknown-linux-gnu"
203    } else if cfg!(all(target_arch = "x86_64", target_os = "windows")) {
204        "x86_64-pc-windows-msvc"
205    } else {
206        "unknown"
207    }
208};
209
210const fn aes_backend_name() -> &'static str {
211    if cfg!(aes_armv8) {
212        "armv8 hardware"
213    } else if cfg!(target_arch = "aarch64") {
214        // Software fallback is the dangerous case this whole mechanism
215        // exists to surface — flag it clearly so `mnemo info` makes it
216        // easy to notice a build picked up the slow path by mistake.
217        "armv8 software (constant-time fallback)"
218    } else if cfg!(target_arch = "x86_64") {
219        // `aes` 0.8.4 on x86_64 picks AES-NI vs the software path at
220        // runtime via the `cpufeatures` crate, so we can't tell from
221        // cfg alone which path a given invocation takes. In practice
222        // every x86_64 CPU shipped since ~2010 has AES-NI.
223        "x86_64 AES-NI (runtime detect)"
224    } else {
225        "portable software"
226    }
227}
228
229const fn polyval_backend_name() -> &'static str {
230    if cfg!(polyval_armv8) {
231        "armv8 PMULL hardware"
232    } else if cfg!(target_arch = "aarch64") {
233        "armv8 software (constant-time fallback)"
234    } else if cfg!(target_arch = "x86_64") {
235        "x86_64 CLMUL (runtime detect)"
236    } else {
237        "portable software"
238    }
239}
240
241/// Test-only crash-injection surface used by the Phase 1.3 proptest
242/// (`mnemo/tests/proptest_store.rs`). Compiled in under `cfg(test)`
243/// and the `failpoints` feature only; absent from release builds.
244///
245/// **Not part of the stable public API.** The double-underscore
246/// convention makes the "internal" contract obvious at the call site.
247#[cfg(any(test, feature = "failpoints"))]
248#[doc(hidden)]
249pub mod __failpoints {
250    /// Arm the pager and WAL commit path to fail on the Nth
251    /// subsequent write. `n = 0` fails the very next write; `n = -1`
252    /// (or any negative) disables. One-shot — disarms on trigger.
253    ///
254    /// Every write is checked before the byte hits the file, so a
255    /// triggered failure is guaranteed torn-BEFORE, never
256    /// torn-AFTER — critical for the "either pre-flush or post-flush"
257    /// invariant the property test asserts.
258    pub fn set_writes_until_fail(n: i64) {
259        crate::pager::failpoints::set_writes_until_fail(n)
260    }
261}
262
263/// WAL entry points for the PERF-1 allocation test
264/// (`tests/wal_recover_alloc.rs`), which needs its own global allocator and
265/// so can't live in this `forbid(unsafe_code)` crate. Not a stable API.
266#[doc(hidden)]
267pub mod __wal {
268    /// Page size of the WAL region.
269    pub const PAGE_SIZE: usize = crate::format::PAGE_SIZE;
270
271    /// One WAL frame: a home page number and its page image.
272    pub type Frame = (u64, Vec<u8>);
273
274    /// Bytes a transaction of `n` page frames occupies.
275    pub fn txn_byte_len(n: usize) -> u64 {
276        crate::wal::txn_byte_len(n, PAGE_SIZE)
277    }
278
279    /// Commit `frames` as transaction `txn_id` at the start of a region.
280    pub fn commit(
281        file: &mut std::fs::File,
282        wal_pages: u64,
283        txn_id: u64,
284        frames: &[Frame],
285    ) -> crate::Result<()> {
286        crate::wal::commit(file, 0, wal_pages, txn_id, frames)
287    }
288
289    /// Run open-time recovery over a region starting at page 0; returns the
290    /// frames to replay, if any.
291    pub fn recover(
292        file: &mut std::fs::File,
293        wal_pages: u64,
294        wal_seq: u64,
295    ) -> crate::Result<Option<Vec<Frame>>> {
296        crate::wal::recover(file, 0, wal_pages, wal_seq)
297    }
298}
299
300/// Internal parse surfaces exposed for cargo-fuzz targets (Phase 1.2).
301///
302/// **Not part of the stable public API.** Each function feeds an
303/// arbitrary byte slice into a parse/decode path that a hostile
304/// `.mnemo` file (or a hostile MCP client) can send us, and returns
305/// `Result<()>` so a fuzz target only asserts "no panic." Marked
306/// `#[doc(hidden)]` so `cargo doc` doesn't advertise it; do not import
307/// from application code.
308#[doc(hidden)]
309pub mod __fuzz {
310    /// Parse a `.mnemo` header page (page 0, plaintext) from arbitrary
311    /// bytes. This is the pre-passphrase attack surface — a hostile
312    /// file hits it before any DEK exists.
313    pub fn parse_header(bytes: &[u8]) -> crate::Result<()> {
314        crate::format::Header::from_page(bytes).map(|_| ())
315    }
316
317    /// Replay the WAL scan over an arbitrary byte region. Delegates
318    /// to [`crate::wal::recover_bytes`]; a corrupt / truncated /
319    /// adversarial region must return `Err` or `Ok(None/Some(_))`,
320    /// never panic.
321    pub fn wal_recover_bytes(region: &[u8], wal_seq: u64) -> crate::Result<()> {
322        crate::wal::recover_bytes(region, wal_seq).map(|_| ())
323    }
324
325    /// Decode a `Memory` record body from arbitrary bytes.
326    pub fn decode_memory(bytes: &[u8]) -> crate::Result<()> {
327        rmp_serde::from_slice::<crate::Memory>(bytes)
328            .map(|_| ())
329            .map_err(|e| crate::MnemoError::Serialize(e.to_string()))
330    }
331
332    /// FMT-9e: decode a v9 `Memory` record body from arbitrary bytes: a
333    /// zstd frame (a compressed thread turn) is decompressed first, under
334    /// the same content-size bound the store applies.
335    pub fn decode_v9_memory(bytes: &[u8]) -> crate::Result<()> {
336        let plain = crate::store::plain_record(bytes)?;
337        rmp_serde::from_slice::<crate::Memory>(&plain)
338            .map(|_| ())
339            .map_err(|e| crate::MnemoError::Serialize(e.to_string()))
340    }
341
342    /// Decode a `CacheEntry` record body from arbitrary bytes. Type is
343    /// `pub(crate)`, so this wrapper hides it behind a unit return.
344    pub fn decode_cache_entry(bytes: &[u8]) -> crate::Result<()> {
345        rmp_serde::from_slice::<crate::result_cache::CacheEntry>(bytes)
346            .map(|_| ())
347            .map_err(|e| crate::MnemoError::Serialize(e.to_string()))
348    }
349
350    /// Decode a cache-directory page-run body (a `Vec<CacheDirectoryEntry>`)
351    /// from arbitrary bytes.
352    pub fn decode_cache_directory(bytes: &[u8]) -> crate::Result<()> {
353        rmp_serde::from_slice::<Vec<crate::result_cache::CacheDirectoryEntry>>(bytes)
354            .map(|_| ())
355            .map_err(|e| crate::MnemoError::Serialize(e.to_string()))
356    }
357
358    /// Parse a single MCP stdio JSON-RPC request line. Malformed
359    /// input returns `Err(String)`; the fuzz target only asserts
360    /// no panic and that the server would not die on this line.
361    pub fn parse_mcp_request_line(line: &str) -> Result<(), String> {
362        crate::mcp::__fuzz_parse_request_line(line)
363    }
364}