nodedb_wal/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! # nodedb-wal
4//!
5//! Deterministic, O_DIRECT write-ahead log with group commit.
6//!
7//! This crate bypasses the Linux page cache entirely. Every WAL write goes
8//! directly to NVMe via `O_DIRECT` (and eventually `io_uring`). This is
9//! non-negotiable: if AI agents dump 10 GB of telemetry logs, the OS must NOT
10//! evict hot HNSW vector indexes from RAM to cache WAL pages.
11//!
12//! ## Design
13//!
14//! - **O_DIRECT**: All writes bypass the page cache. Aligned to 4 KiB.
15//! - **Group commit**: Thousands of concurrent writes are batched into a single
16//!   `fsync`, maximizing NVMe IOPS.
17//! - **CRC32C**: Every record has a checksum for silent bit-rot detection.
18//! - **Deterministic replay**: WAL replay is idempotent — crash at any point,
19//!   recover to a consistent prefix.
20//!
21//! ## Validation target
22//!
23//! Sustain 100,000+ async writes/sec with sub-millisecond p99 latency.
24//! `free -m` cached memory must not move during the benchmark.
25
26pub mod align;
27pub mod crypto;
28pub mod double_write;
29pub mod error;
30pub mod group_commit;
31pub mod lazy_reader;
32#[cfg(not(target_arch = "wasm32"))]
33pub mod mmap_reader;
34pub mod preamble;
35pub mod reader;
36pub mod record;
37pub mod recovery;
38pub mod replay;
39pub mod secure_mem;
40pub mod segment;
41pub mod segmented;
42pub mod temporal_purge;
43pub mod tombstone;
44#[cfg(all(feature = "io-uring", target_os = "linux"))]
45pub mod uring_writer;
46pub mod writer;
47
48pub use double_write::{DoubleWriteBuffer, DwbMode, wal_dwb_bytes_written_total};
49pub use error::{Result, WalError};
50pub use group_commit::GroupCommitter;
51pub use lazy_reader::LazyWalReader;
52pub use preamble::{
53    CIPHER_AES_256_GCM, PREAMBLE_SIZE, PREAMBLE_VERSION, SEG_PREAMBLE_MAGIC, SegmentPreamble,
54    WAL_PREAMBLE_MAGIC,
55};
56pub use record::{
57    CalvinAppliedPayload, FtsDeletePayload, FtsIndexPayload, RecordHeader, RecordType,
58    SpatialDeletePayload, SpatialPutPayload, WalRecord, WalRecordArgs,
59};
60pub use recovery::{RecoveryInfo, recover};
61pub use replay::{DatabaseTombstones, TombstoneSet, extract_tombstones};
62pub use secure_mem::SecureKey;
63pub use segmented::{SegmentedWal, SegmentedWalConfig};
64pub use temporal_purge::{TemporalPurgeEngine, TemporalPurgePayload};
65pub use tombstone::{CollectionTombstonePayload, MAX_COLLECTION_NAME_LEN};
66pub use writer::WalWriter;
nodedb_wal/lib.rs

nodedb_wal/
lib.rs
