nodedb_wal/lib.rs
1//! # nodedb-wal
2//!
3//! Deterministic, O_DIRECT write-ahead log with group commit.
4//!
5//! This crate bypasses the Linux page cache entirely. Every WAL write goes
6//! directly to NVMe via `O_DIRECT` (and eventually `io_uring`). This is
7//! non-negotiable: if AI agents dump 10 GB of telemetry logs, the OS must NOT
8//! evict hot HNSW vector indexes from RAM to cache WAL pages.
9//!
10//! ## Design
11//!
12//! - **O_DIRECT**: All writes bypass the page cache. Aligned to 4 KiB.
13//! - **Group commit**: Thousands of concurrent writes are batched into a single
14//! `fsync`, maximizing NVMe IOPS.
15//! - **CRC32C**: Every record has a checksum for silent bit-rot detection.
16//! - **Deterministic replay**: WAL replay is idempotent — crash at any point,
17//! recover to a consistent prefix.
18//!
19//! ## Validation target
20//!
21//! Sustain 100,000+ async writes/sec with sub-millisecond p99 latency.
22//! `free -m` cached memory must not move during the benchmark.
23
24pub mod align;
25pub mod crypto;
26pub mod double_write;
27pub mod error;
28pub mod group_commit;
29pub mod lazy_reader;
30pub mod mmap_reader;
31pub mod reader;
32pub mod record;
33pub mod recovery;
34pub mod segment;
35pub mod segmented;
36#[cfg(feature = "io-uring")]
37pub mod uring_writer;
38pub mod writer;
39
40pub use error::{Result, WalError};
41pub use group_commit::GroupCommitter;
42pub use lazy_reader::LazyWalReader;
43pub use record::{RecordHeader, RecordType, WalRecord};
44pub use recovery::{RecoveryInfo, recover};
45pub use segmented::{SegmentedWal, SegmentedWalConfig};
46pub use writer::WalWriter;