nodedb_wal/
lib.rs

1// SPDX-License-Identifier: BUSL-1.1
2
3//! # nodedb-wal
4//!
5//! Deterministic, O_DIRECT write-ahead log with group commit.
6//!
7//! This crate bypasses the Linux page cache entirely. Every WAL write goes
8//! directly to NVMe via `O_DIRECT` (and eventually `io_uring`). This is
9//! non-negotiable: if AI agents dump 10 GB of telemetry logs, the OS must NOT
10//! evict hot HNSW vector indexes from RAM to cache WAL pages.
11//!
12//! ## Design
13//!
14//! - **O_DIRECT**: All writes bypass the page cache. Aligned to 4 KiB.
15//! - **Group commit**: Thousands of concurrent writes are batched into a single
16//!   `fsync`, maximizing NVMe IOPS.
17//! - **CRC32C**: Every record has a checksum for silent bit-rot detection.
18//! - **Deterministic replay**: WAL replay is idempotent — crash at any point,
19//!   recover to a consistent prefix.
20//!
21//! ## Validation target
22//!
23//! Sustain 100,000+ async writes/sec with sub-millisecond p99 latency.
24//! `free -m` cached memory must not move during the benchmark.
25
26pub mod align;
27pub mod crypto;
28pub mod double_write;
29pub mod error;
30pub mod group_commit;
31pub mod lazy_reader;
32pub mod mmap_reader;
33pub mod preamble;
34pub mod reader;
35pub mod record;
36pub mod recovery;
37pub mod replay;
38pub mod secure_mem;
39pub mod segment;
40pub mod segmented;
41pub mod temporal_purge;
42pub mod tombstone;
43#[cfg(feature = "io-uring")]
44pub mod uring_writer;
45pub mod writer;
46
47pub use double_write::{DoubleWriteBuffer, DwbMode, wal_dwb_bytes_written_total};
48pub use error::{Result, WalError};
49pub use group_commit::GroupCommitter;
50pub use lazy_reader::LazyWalReader;
51pub use preamble::{
52    CIPHER_AES_256_GCM, PREAMBLE_SIZE, PREAMBLE_VERSION, SEG_PREAMBLE_MAGIC, SegmentPreamble,
53    WAL_PREAMBLE_MAGIC,
54};
55pub use record::{CalvinAppliedPayload, RecordHeader, RecordType, WalRecord};
56pub use recovery::{RecoveryInfo, recover};
57pub use replay::{TombstoneSet, extract_tombstones};
58pub use secure_mem::SecureKey;
59pub use segmented::{SegmentedWal, SegmentedWalConfig};
60pub use temporal_purge::{TemporalPurgeEngine, TemporalPurgePayload};
61pub use tombstone::{CollectionTombstonePayload, MAX_COLLECTION_NAME_LEN};
62pub use writer::WalWriter;
nodedb_wal/lib.rs

nodedb_wal/
lib.rs
