nodedb_wal/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! # nodedb-wal
4//!
5//! Deterministic, O_DIRECT write-ahead log with group commit.
6//!
7//! This crate bypasses the Linux page cache entirely. Every WAL write goes
8//! directly to NVMe via `O_DIRECT` (and eventually `io_uring`). This is
9//! non-negotiable: if AI agents dump 10 GB of telemetry logs, the OS must NOT
10//! evict hot HNSW vector indexes from RAM to cache WAL pages.
11//!
12//! ## Design
13//!
14//! - **O_DIRECT**: All writes bypass the page cache. Aligned to 4 KiB.
15//! - **Group commit**: Thousands of concurrent writes are batched into a single
16//!   `fsync`, maximizing NVMe IOPS.
17//! - **CRC32C**: Every record has a checksum for silent bit-rot detection.
18//! - **Deterministic replay**: WAL replay is idempotent — crash at any point,
19//!   recover to a consistent prefix.
20//!
21//! ## Validation target
22//!
23//! Sustain 100,000+ async writes/sec with sub-millisecond p99 latency.
24//! `free -m` cached memory must not move during the benchmark.
25//!
26//! ## Targets
27//!
28//! On wasm32 the crate builds only [`crypto`], the key and envelope code it
29//! needs, [`error`], and the record-header constants in [`record::header`].
30//! The writer, readers, segments, double-write buffer, replay, and recovery
31//! build only on native targets.
32
33#[cfg(not(target_arch = "wasm32"))]
34pub mod align;
35pub mod crypto;
36#[cfg(not(target_arch = "wasm32"))]
37pub mod diag;
38#[cfg(not(target_arch = "wasm32"))]
39pub mod double_write;
40pub mod error;
41#[cfg(not(target_arch = "wasm32"))]
42pub mod lazy_reader;
43#[cfg(not(target_arch = "wasm32"))]
44pub mod mmap_reader;
45#[cfg(not(target_arch = "wasm32"))]
46pub mod preamble;
47#[cfg(not(target_arch = "wasm32"))]
48pub mod reader;
49pub mod record;
50#[cfg(not(target_arch = "wasm32"))]
51pub mod recovery;
52#[cfg(not(target_arch = "wasm32"))]
53pub mod replay;
54pub mod secure_mem;
55#[cfg(not(target_arch = "wasm32"))]
56pub mod segment;
57mod segment_envelope;
58#[cfg(not(target_arch = "wasm32"))]
59pub mod segmented;
60#[cfg(not(target_arch = "wasm32"))]
61pub mod temporal_purge;
62#[cfg(not(target_arch = "wasm32"))]
63pub mod time_anchors;
64#[cfg(not(target_arch = "wasm32"))]
65pub mod tombstone;
66#[cfg(not(target_arch = "wasm32"))]
67pub mod torn_tail;
68#[cfg(all(feature = "io-uring", target_os = "linux"))]
69pub mod uring_writer;
70#[cfg(not(target_arch = "wasm32"))]
71pub mod writer;
72
73#[cfg(not(target_arch = "wasm32"))]
74pub use double_write::{
75    DoubleWriteBuffer, DwbDegradation, DwbMirror, DwbMode, DwbProtection, DwbSkipReason,
76    wal_dwb_bytes_written_total, wal_dwb_degradations_total, wal_dwb_unprotected_records_total,
77};
78pub use error::{Result, WalError};
79#[cfg(not(target_arch = "wasm32"))]
80pub use lazy_reader::LazyWalReader;
81#[cfg(not(target_arch = "wasm32"))]
82pub use preamble::{
83    CIPHER_AES_256_GCM, PREAMBLE_SIZE, PREAMBLE_VERSION, SEG_PREAMBLE_MAGIC, SegmentPreamble,
84    WAL_PREAMBLE_MAGIC,
85};
86#[cfg(not(target_arch = "wasm32"))]
87pub use reader::{StopReason, WalReader};
88#[cfg(not(target_arch = "wasm32"))]
89pub use record::{
90    FtsDeletePayload, FtsIndexPayload, RecordStamp, RecordTarget, RecordType, SpatialDeletePayload,
91    SpatialPutPayload, WalRecord, WalRecordArgs, WriteAbortedPayload,
92};
93pub use record::{NO_EVENT_SOURCE, RecordHeader};
94#[cfg(not(target_arch = "wasm32"))]
95pub use recovery::{RecoveryInfo, recover};
96#[cfg(not(target_arch = "wasm32"))]
97pub use replay::{
98    AbortedWrites, DatabaseTombstones, ReplayFilters, TombstoneSet, drop_aborted_records,
99    extract_replay_filters, extract_tombstones,
100};
101pub use secure_mem::SecureKey;
102#[cfg(not(target_arch = "wasm32"))]
103pub use segmented::{SegmentedWal, SegmentedWalConfig};
104#[cfg(not(target_arch = "wasm32"))]
105pub use temporal_purge::{TemporalPurgeEngine, TemporalPurgePayload};
106#[cfg(not(target_arch = "wasm32"))]
107pub use time_anchors::TimeAnchors;
108#[cfg(not(target_arch = "wasm32"))]
109pub use tombstone::{CollectionTombstonePayload, MAX_COLLECTION_NAME_LEN};
110#[cfg(not(target_arch = "wasm32"))]
111pub use torn_tail::{TailVerdict, holds_only_crash_residue, verify_committed_prefix};
112#[cfg(not(target_arch = "wasm32"))]
113pub use writer::WalWriter;
nodedb_wal/lib.rs

nodedb_wal/
lib.rs
