mentedb_storage/lib.rs
1//! MenteDB Storage Engine: compact record storage with WAL and record cache.
2//!
3//! This crate implements the low-level storage engine:
4//! - **Record file**: variable length memory records packed into 512 byte
5//! blocks, so the file is about the size of the data
6//! - **Locator table**: stable record ids resolved in memory, persisted at
7//! checkpoints and replayed forward from the WAL on open
8//! - **Write-ahead log (WAL)**: append-only, CRC-checked, LZ4-compressed
9//! entries that redo every record write byte for byte
10//! - **Record cache**: decoded records bounded by a configurable byte budget
11//! - **Migration**: databases in the old 64 KB page layout are compacted into
12//! the record layout on open, atomically
13//!
14//! # Concurrency
15//!
16//! One process per database directory, enforced by an exclusive lock on a
17//! LOCK file (fcntl OFD lock, honored across hosts on NFSv4). Writes are
18//! serialized inside the process; reads run concurrently.
19//!
20//! # Example
21//!
22//! ```no_run
23//! use mentedb_storage::StorageEngine;
24//! use mentedb_core::{MemoryNode, memory::MemoryType, types::AgentId};
25//!
26//! let engine = StorageEngine::open("/tmp/mentedb-data".as_ref())?;
27//!
28//! let node = MemoryNode::new(
29//! AgentId::new(),
30//! MemoryType::Episodic,
31//! "The user prefers Rust".to_string(),
32//! vec![0.1, 0.2, 0.3],
33//! );
34//!
35//! let page_id = engine.store_memory(&node)?;
36//! let loaded = engine.load_memory(page_id)?;
37//! assert_eq!(loaded.content, "The user prefers Rust");
38//!
39//! engine.checkpoint()?;
40//! engine.close()?;
41//! # Ok::<(), mentedb_core::error::MenteError>(())
42//! ```
43
44/// Snapshot backup utilities.
45pub mod backup;
46/// Byte bounded cache of decoded records with CLOCK eviction.
47pub mod cache;
48/// Unified storage facade for memory persistence.
49pub mod engine;
50/// Migration from the 64 KB page file layout.
51pub mod legacy;
52/// Cross-host single-writer lock (fcntl OFD locks on Linux/EFS).
53mod lock;
54/// Persisted locator table, replayed forward from the WAL on open.
55pub mod memory_map;
56/// Record file format: extents of 512 byte blocks, payload codec, allocator.
57pub mod record;
58/// Append only write ahead log with CRC checks and LZ4 compression.
59pub mod wal;
60
61// Re-export key types at crate root for convenience.
62pub use cache::{BufferStats, RecordCache};
63pub use engine::StorageEngine;
64pub use legacy::MigrationReport;
65pub use memory_map::MappedMemory;
66pub use record::{BLOCK_SIZE, PageId};
67pub use wal::{Lsn, Wal, WalEntry, WalEntryType};