Skip to main content

mnemo_engine/
lib.rs

1//! # Mnemo
2//!
3//! Mnemo is an **encrypted, single-file, portable agent-memory engine**.
4//!
5//! A whole memory store — vectors, content, metadata, and the multi-signal
6//! recall machinery an agent needs — lives in one file you can copy, back up,
7//! or hand to another process. The file is encrypted at rest with a two-tier
8//! key hierarchy: an Argon2id key-encryption key (KEK) derived from a
9//! passphrase wraps a random data-encryption key (DEK), and the DEK encrypts
10//! every page with AES-256-GCM.
11//!
12//! ## Quick start
13//!
14//! ```no_run
15//! use mnemo_engine::{Mnemo, MnemoConfig, Memory, MemoryType, RecallRequest};
16//!
17//! # fn main() -> mnemo_engine::Result<()> {
18//! let cfg = MnemoConfig { dimensions: 3, ..Default::default() };
19//! let mut db = Mnemo::create("agent.mnemo", "correct horse battery", cfg)?;
20//!
21//! db.remember(
22//!     Memory::new("the user prefers dark mode", MemoryType::Semantic, vec![0.1, 0.2, 0.9])
23//!         .with_agent("assistant-1")
24//!         .with_importance(0.8),
25//! )?;
26//! db.flush()?;
27//!
28//! let hits = db.recall(&RecallRequest::new(vec![0.1, 0.2, 0.9]).top_k(5))?;
29//! for h in hits {
30//!     println!("{:.3}  {}", h.score, h.memory.content);
31//! }
32//! # Ok(())
33//! # }
34//! ```
35//!
36//! ## What is and is not built
37//!
38//! This crate implements a real, tested core: the encrypted single-file
39//! storage engine, the crypto layer, the agent-memory model, multi-signal
40//! recall, an IVF+PQ approximate-nearest-neighbour index, a write-ahead log,
41//! snapshot-based point-in-time recovery, a bounded LRU page cache, and the
42//! [`Session`] conversation wrapper. Exact brute-force search remains
43//! available as the ground-truth baseline; a built index makes
44//! [`Mnemo::recall`] sub-linear. Each [`Mnemo::flush`] is one atomic,
45//! WAL-committed transaction and a restorable snapshot — [`Mnemo::restore_to`]
46//! rewinds the database to any past transaction. Python and TypeScript
47//! language bindings are the one documented roadmap item — see the README.
48
49#![forbid(unsafe_code)]
50#![warn(missing_docs)]
51
52mod cache;
53mod crypto;
54mod embed;
55mod embed_cmd;
56mod error;
57mod format;
58mod freelist;
59mod index;
60mod keyslot;
61pub mod mcp;
62mod memory;
63mod pager;
64mod result_cache;
65mod session;
66mod store;
67mod text;
68mod transcript;
69mod veccol;
70mod wal;
71
72pub use crypto::KdfParams;
73pub use embed::{Embedder, EmbedderIdentity};
74pub use embed_cmd::{EmbedCmd, DEFAULT_EMBED_TIMEOUT};
75pub use error::{MnemoError, Result};
76pub use index::{IndexConfig, IndexInfo};
77pub use memory::{
78    status as essence_status, CommitBatch, CommitReport, EssenceOp, EssenceOpKind, EssenceScope,
79    EssenceSection, ExportedTurn, Filter, FilterTerm, IngestReport, KeyScope, KeyedMemory, Memory,
80    MemoryPatch, MemoryType, Metric, Scope, ScoreWeights, ThreadTurn, TurnPolicy, WriteOutcome,
81    CONTINUATION_TIMELINE_ENTRIES, ESSENCE_DEFAULT_RENDER_BYTES, ESSENCE_SECTION_MAX_BYTES,
82    ESSENCE_TIMELINE_MAX_BYTES, ESSENCE_TRANSITIONS, FINDING_TRANSITIONS, MNEMO_META_CONTINUES,
83    MNEMO_META_INVALID_AT, MNEMO_META_KEY, MNEMO_META_KIND, MNEMO_META_KIND_ESSENCE,
84    MNEMO_META_KIND_THREAD_LINK, MNEMO_META_KIND_TURN, MNEMO_META_LENGTH, MNEMO_META_NS,
85    MNEMO_META_PART, MNEMO_META_REFS, MNEMO_META_ROLE, MNEMO_META_SECTION, MNEMO_META_SEQ,
86    MNEMO_META_SHA256, MNEMO_META_STATUS, MNEMO_META_STATUS_HISTORY, MNEMO_META_SUPERSEDED_BY,
87    MNEMO_META_SUPERSEDES, MNEMO_META_THREAD, MNEMO_META_TURN_ID, MNEMO_META_UPDATED_AT,
88    TASK_TRANSITIONS, THREAD_TOOL_RESULT_TRUNCATE_BYTES, THREAD_TURN_CHUNK_BYTES,
89};
90pub use result_cache::{
91    CacheBudget, CacheFlushPolicy, CachePutOpts, CacheStats, CachedValue, SemanticCachePutOpts,
92    DEFAULT_BATCH_MAX_AGE, DEFAULT_BATCH_MAX_DIRTY, DEFAULT_MAX_BYTES, DEFAULT_MAX_ENTRIES,
93    DEFAULT_SEMANTIC_THRESHOLD,
94};
95pub use session::{Role, Session, Turn};
96#[doc(hidden)]
97pub use store::BudgetReport;
98pub use store::{
99    CompactReport, Mnemo, MnemoConfig, RecallRequest, RecallResponse, RecallResult, SnapshotInfo,
100    Stats, DEFAULT_KEYWORD_WEIGHT, DEFAULT_TURN_COMPRESS_AFTER_SECS, DEFAULT_VECTOR_CACHE_BUDGET,
101    INDEX_PERSIST_EVERY_CHANGES, INDEX_PERSIST_EVERY_COMMITS, V9_ANN_MIN_FACTS, V9_RERANK_PER_HIT,
102};
103pub use transcript::{read_transcript, TranscriptFormat};
104
105/// Re-export of [`ulid::Ulid`], the identifier type used for memories.
106pub use ulid::Ulid;
107
108// --- Fsync counter -----------------------------------------------------
109//
110// Instrumentation plumbing for the Phase 6 top-3 cost #2 fix: the
111// planner wants to know how many `sync_all` calls each `flush()` issues
112// before deciding whether to batch any of them. Every fsync call site
113// bumps a thread-local counter; `Mnemo::flush` snapshots it before and
114// after, exposing the delta on `Stats::fsyncs_last_flush`. Thread-local
115// so handles on separate threads don't poison each other's counts; the
116// engine's one-writer-at-a-time locking means a single handle sees a
117// coherent stream.
118
119mod fsync_counter {
120    use std::cell::Cell;
121
122    thread_local! {
123        static COUNT: Cell<u64> = const { Cell::new(0) };
124    }
125
126    /// Call once per `sync_all`.
127    pub(crate) fn bump() {
128        COUNT.with(|c| c.set(c.get().wrapping_add(1)));
129    }
130
131    /// Monotonic counter snapshot for this thread.
132    pub(crate) fn count() -> u64 {
133        COUNT.with(|c| c.get())
134    }
135}
136
137pub(crate) use fsync_counter::{bump as bump_fsync, count as fsync_count};
138
139/// Build-time cryptographic backend information.
140///
141/// Populated at compile time from the active `cfg` flags. On ARMv8
142/// builds compiled with `--cfg aes_armv8 --cfg polyval_armv8` (set by
143/// the repo's `.cargo/config.toml` and both CI workflows) the `aes`
144/// and `polyval` crates emit hardware AES + PMULL instructions; without
145/// those flags they fall back to a constant-time software path that
146/// runs at roughly 5× the cost per page.
147///
148/// Published through [`build_info`], exposed by `mnemo info` on the
149/// CLI and `mnemo.build_info()` from the Python binding. The point is
150/// verification — nobody should have to profile a release build to find
151/// out whether it's running software AES.
152#[derive(Debug, Clone, Copy, PartialEq, Eq)]
153pub struct BuildInfo {
154    /// Package version (`CARGO_PKG_VERSION`), e.g. `"0.5.1"`.
155    pub version: &'static str,
156    /// Target triple this binary was built for, e.g.
157    /// `"aarch64-apple-darwin"`.
158    pub target: &'static str,
159    /// Human-readable AES backend. One of:
160    /// - `"armv8 hardware"` — ARM with `cfg(aes_armv8)`
161    /// - `"armv8 software"` — ARM without the cfg; constant-time fallback
162    /// - `"x86_64 AES-NI (runtime detect)"` — x86_64 with runtime
163    ///   AES-NI detection in the `aes` crate
164    /// - `"portable software"` — any other target
165    pub aes_backend: &'static str,
166    /// Human-readable PMULL/CLMUL backend used by GHASH (part of AES-GCM).
167    /// Same shape as `aes_backend` but for `polyval`.
168    pub polyval_backend: &'static str,
169}
170
171/// Report this build's cryptographic backend configuration.
172///
173/// Cheap — all fields are `&'static str` populated at compile time.
174/// Safe to call from a hot path if, for some reason, that's useful.
175pub fn build_info() -> BuildInfo {
176    BuildInfo {
177        version: env!("CARGO_PKG_VERSION"),
178        target: TARGET_TRIPLE,
179        aes_backend: aes_backend_name(),
180        polyval_backend: polyval_backend_name(),
181    }
182}
183
184/// Target triple baked in via the `TARGET` env var that `cargo` sets
185/// for the host compile. Falls back to `"unknown"` if the var isn't
186/// set at build time (shouldn't happen under cargo, but a build.rs-free
187/// implementation keeps this crate build-script-free, so we use a
188/// cfg-based dispatch as the second-best).
189const TARGET_TRIPLE: &str = {
190    // `env!("TARGET")` isn't set by cargo for crates; it's set by rustc
191    // only for `std` builds. We approximate from `target_arch` /
192    // `target_os` cfgs so this stays build-script-free. Good enough
193    // for an info line; precise triples are a `cargo` or build.rs job.
194    if cfg!(all(target_arch = "aarch64", target_os = "macos")) {
195        "aarch64-apple-darwin"
196    } else if cfg!(all(target_arch = "aarch64", target_os = "linux")) {
197        "aarch64-unknown-linux-gnu"
198    } else if cfg!(all(target_arch = "x86_64", target_os = "macos")) {
199        "x86_64-apple-darwin"
200    } else if cfg!(all(target_arch = "x86_64", target_os = "linux")) {
201        "x86_64-unknown-linux-gnu"
202    } else if cfg!(all(target_arch = "x86_64", target_os = "windows")) {
203        "x86_64-pc-windows-msvc"
204    } else {
205        "unknown"
206    }
207};
208
209const fn aes_backend_name() -> &'static str {
210    if cfg!(aes_armv8) {
211        "armv8 hardware"
212    } else if cfg!(target_arch = "aarch64") {
213        // Software fallback is the dangerous case this whole mechanism
214        // exists to surface — flag it clearly so `mnemo info` makes it
215        // easy to notice a build picked up the slow path by mistake.
216        "armv8 software (constant-time fallback)"
217    } else if cfg!(target_arch = "x86_64") {
218        // `aes` 0.8.4 on x86_64 picks AES-NI vs the software path at
219        // runtime via the `cpufeatures` crate, so we can't tell from
220        // cfg alone which path a given invocation takes. In practice
221        // every x86_64 CPU shipped since ~2010 has AES-NI.
222        "x86_64 AES-NI (runtime detect)"
223    } else {
224        "portable software"
225    }
226}
227
228const fn polyval_backend_name() -> &'static str {
229    if cfg!(polyval_armv8) {
230        "armv8 PMULL hardware"
231    } else if cfg!(target_arch = "aarch64") {
232        "armv8 software (constant-time fallback)"
233    } else if cfg!(target_arch = "x86_64") {
234        "x86_64 CLMUL (runtime detect)"
235    } else {
236        "portable software"
237    }
238}
239
240/// Test-only crash-injection surface used by the Phase 1.3 proptest
241/// (`mnemo/tests/proptest_store.rs`). Compiled in under `cfg(test)`
242/// and the `failpoints` feature only; absent from release builds.
243///
244/// **Not part of the stable public API.** The double-underscore
245/// convention makes the "internal" contract obvious at the call site.
246#[cfg(any(test, feature = "failpoints"))]
247#[doc(hidden)]
248pub mod __failpoints {
249    /// Arm the pager and WAL commit path to fail on the Nth
250    /// subsequent write. `n = 0` fails the very next write; `n = -1`
251    /// (or any negative) disables. One-shot — disarms on trigger.
252    ///
253    /// Every write is checked before the byte hits the file, so a
254    /// triggered failure is guaranteed torn-BEFORE, never
255    /// torn-AFTER — critical for the "either pre-flush or post-flush"
256    /// invariant the property test asserts.
257    pub fn set_writes_until_fail(n: i64) {
258        crate::pager::failpoints::set_writes_until_fail(n)
259    }
260}
261
262/// WAL entry points for the PERF-1 allocation test
263/// (`tests/wal_recover_alloc.rs`), which needs its own global allocator and
264/// so can't live in this `forbid(unsafe_code)` crate. Not a stable API.
265#[doc(hidden)]
266pub mod __wal {
267    /// Page size of the WAL region.
268    pub const PAGE_SIZE: usize = crate::format::PAGE_SIZE;
269
270    /// One WAL frame: a home page number and its page image.
271    pub type Frame = (u64, Vec<u8>);
272
273    /// Bytes a transaction of `n` page frames occupies.
274    pub fn txn_byte_len(n: usize) -> u64 {
275        crate::wal::txn_byte_len(n, PAGE_SIZE)
276    }
277
278    /// Commit `frames` as transaction `txn_id` at the start of a region.
279    pub fn commit(
280        file: &mut std::fs::File,
281        wal_pages: u64,
282        txn_id: u64,
283        frames: &[Frame],
284    ) -> crate::Result<()> {
285        crate::wal::commit(file, 0, wal_pages, txn_id, frames)
286    }
287
288    /// Run open-time recovery over a region starting at page 0; returns the
289    /// frames to replay, if any.
290    pub fn recover(
291        file: &mut std::fs::File,
292        wal_pages: u64,
293        wal_seq: u64,
294    ) -> crate::Result<Option<Vec<Frame>>> {
295        crate::wal::recover(file, 0, wal_pages, wal_seq)
296    }
297}
298
299/// Internal parse surfaces exposed for cargo-fuzz targets (Phase 1.2).
300///
301/// **Not part of the stable public API.** Each function feeds an
302/// arbitrary byte slice into a parse/decode path that a hostile
303/// `.mnemo` file (or a hostile MCP client) can send us, and returns
304/// `Result<()>` so a fuzz target only asserts "no panic." Marked
305/// `#[doc(hidden)]` so `cargo doc` doesn't advertise it; do not import
306/// from application code.
307#[doc(hidden)]
308pub mod __fuzz {
309    /// Parse a `.mnemo` header page (page 0, plaintext) from arbitrary
310    /// bytes. This is the pre-passphrase attack surface — a hostile
311    /// file hits it before any DEK exists.
312    pub fn parse_header(bytes: &[u8]) -> crate::Result<()> {
313        crate::format::Header::from_page(bytes).map(|_| ())
314    }
315
316    /// Replay the WAL scan over an arbitrary byte region. Delegates
317    /// to [`crate::wal::recover_bytes`]; a corrupt / truncated /
318    /// adversarial region must return `Err` or `Ok(None/Some(_))`,
319    /// never panic.
320    pub fn wal_recover_bytes(region: &[u8], wal_seq: u64) -> crate::Result<()> {
321        crate::wal::recover_bytes(region, wal_seq).map(|_| ())
322    }
323
324    /// Decode a `Memory` record body from arbitrary bytes.
325    pub fn decode_memory(bytes: &[u8]) -> crate::Result<()> {
326        rmp_serde::from_slice::<crate::Memory>(bytes)
327            .map(|_| ())
328            .map_err(|e| crate::MnemoError::Serialize(e.to_string()))
329    }
330
331    /// FMT-9e: decode a v9 `Memory` record body from arbitrary bytes: a
332    /// zstd frame (a compressed thread turn) is decompressed first, under
333    /// the same content-size bound the store applies.
334    pub fn decode_v9_memory(bytes: &[u8]) -> crate::Result<()> {
335        let plain = crate::store::plain_record(bytes)?;
336        rmp_serde::from_slice::<crate::Memory>(&plain)
337            .map(|_| ())
338            .map_err(|e| crate::MnemoError::Serialize(e.to_string()))
339    }
340
341    /// Decode a `CacheEntry` record body from arbitrary bytes. Type is
342    /// `pub(crate)`, so this wrapper hides it behind a unit return.
343    pub fn decode_cache_entry(bytes: &[u8]) -> crate::Result<()> {
344        rmp_serde::from_slice::<crate::result_cache::CacheEntry>(bytes)
345            .map(|_| ())
346            .map_err(|e| crate::MnemoError::Serialize(e.to_string()))
347    }
348
349    /// Decode a cache-directory page-run body (a `Vec<CacheDirectoryEntry>`)
350    /// from arbitrary bytes.
351    pub fn decode_cache_directory(bytes: &[u8]) -> crate::Result<()> {
352        rmp_serde::from_slice::<Vec<crate::result_cache::CacheDirectoryEntry>>(bytes)
353            .map(|_| ())
354            .map_err(|e| crate::MnemoError::Serialize(e.to_string()))
355    }
356
357    /// Parse a single MCP stdio JSON-RPC request line. Malformed
358    /// input returns `Err(String)`; the fuzz target only asserts
359    /// no panic and that the server would not die on this line.
360    pub fn parse_mcp_request_line(line: &str) -> Result<(), String> {
361        crate::mcp::__fuzz_parse_request_line(line)
362    }
363}