1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
use std::sync::Arc;
use zeroize::Zeroizing;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DurabilityMode {
Full,
Batch,
OsBuffered,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum RecoveryMode {
Strict,
Permissive,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum StorageMode {
InMemory,
/// Durable WAL/checkpoint state plus append-only disk storage for KV payloads.
///
/// This mode keeps table rows, indexes, KV keys, versions, and metadata in
/// memory. Set `persistent_value_inline_threshold_bytes` to `0` when KV
/// payloads must not remain inline in memory.
#[default]
DiskBacked,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
pub enum PrimaryIndexBackend {
#[default]
OrdMap,
ArtExperimental,
}
/// Runtime configuration for an AEDB instance.
#[derive(Debug, Clone)]
pub struct AedbConfig {
pub max_segment_bytes: u64,
pub max_segment_age_secs: u64,
pub durability_mode: DurabilityMode,
pub batch_interval_ms: u64,
pub batch_max_bytes: usize,
pub idempotency_window_seconds: u64,
pub idempotency_window_commits: u64,
pub max_inflight_commits: usize,
pub max_commit_queue_bytes: usize,
pub max_transaction_bytes: usize,
/// Maximum number of mutations allowed in a single envelope. Enforced
/// before any work is performed to bound DoS surface from oversized
/// envelopes.
pub max_mutations_per_envelope: usize,
/// Maximum number of read assertions allowed in a single envelope.
/// Enforced before any work is performed.
pub max_read_assertions_per_envelope: usize,
/// Aggregate cap on bytes scanned by reads (mutation Read-Modify-Write
/// scans and assertion evaluation) within a single envelope. Enforced
/// incrementally during read traversal.
pub max_read_bytes_per_envelope: usize,
pub commit_timeout_ms: u64,
pub durable_ack_coalescing_enabled: bool,
pub durable_ack_coalesce_window_us: u64,
pub max_snapshot_age_ms: u64,
pub max_concurrent_snapshots: usize,
pub max_scan_rows: usize,
pub max_batch_rows: usize,
pub max_kv_key_bytes: usize,
pub max_kv_value_bytes: usize,
pub storage_mode: StorageMode,
pub persistent_value_inline_threshold_bytes: usize,
pub persistent_value_hot_cache_bytes: usize,
pub kv_segment_block_cache_bytes: usize,
pub max_table_value_bytes: usize,
pub max_event_payload_bytes: usize,
pub max_memory_estimate_bytes: usize,
pub prestage_shards: usize,
pub epoch_max_wait_us: u64,
pub epoch_min_commits: usize,
pub epoch_max_commits: usize,
pub adaptive_epoch_enabled: bool,
pub adaptive_epoch_min_commits_floor: usize,
pub adaptive_epoch_min_commits_ceiling: usize,
pub adaptive_epoch_wait_us_floor: u64,
pub adaptive_epoch_wait_us_ceiling: u64,
pub adaptive_epoch_target_latency_us: u64,
pub parallel_apply_enabled: bool,
pub parallel_worker_threads: usize,
pub coordinator_locking_enabled: bool,
pub global_unique_index_enabled: bool,
/// When enabled, each broadcast `CommitDelta` carries resolved row-level
/// changes (`CommitDelta::row_changes`) derived from the pre-commit state,
/// so subscription layers can compute per-row diffs without re-querying.
/// Off by default: derivation adds a bounded pre-commit read per committed
/// table transaction, paid only when a consumer needs the feed.
pub row_change_deltas_enabled: bool,
pub partition_lock_timeout_ms: u64,
pub epoch_apply_timeout_ms: u64,
pub max_versions: usize,
pub version_store_full_snapshot_interval_deltas: usize,
pub min_version_age_ms: u64,
pub version_gc_interval_ms: u64,
/// Encryption key for checkpoint files. Wrapped in Arc<Zeroizing<>> to ensure
/// the key is securely zeroed from memory when the last reference is dropped,
/// preventing key disclosure through core dumps or memory scanning.
pub checkpoint_encryption_key: Option<Arc<Zeroizing<[u8; 32]>>>,
pub checkpoint_key_id: Option<String>,
/// zstd compression level used for checkpoint payloads.
/// Typical useful range is 0..=19 where lower is faster and larger.
pub checkpoint_compression_level: i32,
/// Number of most-recent checkpoints to retain in the manifest. Older
/// checkpoints (and the WAL segments that precede the oldest retained one)
/// are eligible for reclamation. Keeping more than one provides a fallback
/// if the newest checkpoint is corrupt, at the cost of extra disk usage.
/// Effective value is clamped to at least 1.
pub checkpoint_retention_count: usize,
/// HMAC key for manifest integrity. Wrapped in Arc<Zeroizing<>> to ensure
/// the key is securely zeroed from memory when the last reference is dropped.
pub manifest_hmac_key: Option<Arc<Zeroizing<Vec<u8>>>>,
/// HMAC-SHA256 key used to sign query pagination cursor tokens so that
/// clients cannot tamper with `last_sort_key`, `last_pk`, `page_size`, or
/// `snapshot_seq`. When `None`, cursors are emitted unsigned (legacy
/// behaviour) for back-compat with existing tests and embedders. When
/// `Some`, both encode and decode paths require a valid 32-byte HMAC tag.
pub cursor_signing_key: Option<Arc<Zeroizing<[u8; 32]>>>,
pub recovery_mode: RecoveryMode,
pub hash_chain_required: bool,
pub primary_index_backend: PrimaryIndexBackend,
/// When enabled (and a persistent value store is attached), table-row
/// payloads are spilled to disk under memory pressure instead of rejecting
/// the commit. Coldest rows spill first and are paged back in on read
/// through the value store's hot cache. Keys, indexes, and row versions
/// remain resident. No effect in `StorageMode::InMemory`.
pub table_row_spill_enabled: bool,
/// When enabled (and a KV segment store is attached), whole cold table rows
/// — key and value — are evicted from the resident `rows` map to sorted
/// on-disk segments under memory pressure, going beyond payload spill to
/// release the resident primary keys too. Evicted rows page back in on read
/// through the segment store and are re-inlined before every checkpoint.
///
/// Off by default: it is only safe to enable once every table-row read path
/// is tier-aware. No effect in `StorageMode::InMemory`.
pub table_row_segment_eviction_enabled: bool,
/// When enabled (and a KV segment store is attached), cold secondary-index
/// postings are evicted from the resident index stores to sorted on-disk
/// segments under memory pressure, so a table's indexes can grow beyond RAM
/// just like its rows. Evicted postings page back in on read through the
/// segment store and are re-inlined before every checkpoint (the index cold
/// tier is runtime-only and never persisted).
///
/// Off by default: it is only safe to enable once every secondary-index read
/// path — equality lookups, ordered range scans, and unique-constraint
/// checks (including the global unique index) — is tier-aware. No effect in
/// `StorageMode::InMemory`.
pub index_segment_eviction_enabled: bool,
/// Capacity of the public commit-delta broadcast channel
/// exposed via `AedbInstance::subscribe_commits`. Subscribers
/// that fall behind beyond this many buffered deltas will
/// receive `RecvError::Lagged` from `broadcast::Receiver::recv`.
pub commit_broadcast_capacity: usize,
}
impl Default for AedbConfig {
fn default() -> Self {
Self {
max_segment_bytes: 64 * 1024 * 1024,
max_segment_age_secs: 60,
durability_mode: DurabilityMode::Full,
batch_interval_ms: 10,
batch_max_bytes: 1024 * 1024,
idempotency_window_seconds: 300,
idempotency_window_commits: 100_000,
max_inflight_commits: 64,
max_commit_queue_bytes: 64 * 1024 * 1024,
max_transaction_bytes: 16 * 1024 * 1024,
max_mutations_per_envelope: 16_384,
max_read_assertions_per_envelope: 32_768,
max_read_bytes_per_envelope: 16 * 1024 * 1024,
commit_timeout_ms: 5000,
durable_ack_coalescing_enabled: false,
durable_ack_coalesce_window_us: 0,
max_snapshot_age_ms: 30_000,
max_concurrent_snapshots: 128,
max_scan_rows: 10_000,
max_batch_rows: 10_000,
max_kv_key_bytes: 1024,
max_kv_value_bytes: 1024 * 1024,
storage_mode: StorageMode::DiskBacked,
persistent_value_inline_threshold_bytes: 64 * 1024,
persistent_value_hot_cache_bytes: 64 * 1024 * 1024,
kv_segment_block_cache_bytes: 16 * 1024 * 1024,
max_table_value_bytes: 1024 * 1024,
max_event_payload_bytes: 64 * 1024,
max_memory_estimate_bytes: 2 * 1024 * 1024 * 1024,
prestage_shards: 8,
epoch_max_wait_us: 100,
epoch_min_commits: 1,
epoch_max_commits: 64,
adaptive_epoch_enabled: true,
adaptive_epoch_min_commits_floor: 1,
adaptive_epoch_min_commits_ceiling: 128,
adaptive_epoch_wait_us_floor: 25,
adaptive_epoch_wait_us_ceiling: 1_000,
adaptive_epoch_target_latency_us: 1_500,
parallel_apply_enabled: true,
parallel_worker_threads: std::thread::available_parallelism()
.map(|n| n.get().max(2))
.unwrap_or(4),
coordinator_locking_enabled: true,
global_unique_index_enabled: true,
row_change_deltas_enabled: false,
partition_lock_timeout_ms: 5_000,
epoch_apply_timeout_ms: 10_000,
max_versions: 1024,
version_store_full_snapshot_interval_deltas: 128,
min_version_age_ms: 5_000,
version_gc_interval_ms: 1_000,
checkpoint_encryption_key: None,
checkpoint_key_id: None,
checkpoint_compression_level: 3,
checkpoint_retention_count: 3,
manifest_hmac_key: None,
cursor_signing_key: None,
recovery_mode: RecoveryMode::Strict,
hash_chain_required: true,
primary_index_backend: PrimaryIndexBackend::OrdMap,
table_row_spill_enabled: true,
table_row_segment_eviction_enabled: false,
index_segment_eviction_enabled: false,
commit_broadcast_capacity: 1024,
}
}
}
impl AedbConfig {
pub fn production(hmac_key: [u8; 32]) -> Self {
Self {
manifest_hmac_key: Some(Arc::new(Zeroizing::new(hmac_key.to_vec()))),
recovery_mode: RecoveryMode::Strict,
durability_mode: DurabilityMode::Full,
hash_chain_required: true,
storage_mode: StorageMode::DiskBacked,
persistent_value_inline_threshold_bytes: 0,
..Self::default()
}
}
pub fn development() -> Self {
Self {
manifest_hmac_key: None,
recovery_mode: RecoveryMode::Permissive,
durability_mode: DurabilityMode::OsBuffered,
hash_chain_required: false,
..Self::default()
}
}
/// Low-latency profile tuned for user-facing commit acknowledgements.
///
/// This profile intentionally uses batched durability so commits can
/// finalize at visible-head latency while WAL fsync is coalesced in short
/// intervals. It keeps strict recovery + hash-chain + manifest HMAC.
///
/// Use `AedbInstance::commit_with_finality(..., CommitFinality::Durable)`
/// for flows that must wait for durability.
pub fn low_latency(hmac_key: [u8; 32]) -> Self {
Self {
manifest_hmac_key: Some(Arc::new(Zeroizing::new(hmac_key.to_vec()))),
recovery_mode: RecoveryMode::Strict,
durability_mode: DurabilityMode::Batch,
hash_chain_required: true,
batch_interval_ms: 2,
batch_max_bytes: 512 * 1024,
durable_ack_coalescing_enabled: true,
durable_ack_coalesce_window_us: 500,
epoch_max_wait_us: 50,
adaptive_epoch_target_latency_us: 1_000,
storage_mode: StorageMode::DiskBacked,
persistent_value_inline_threshold_bytes: 0,
..Self::default()
}
}
pub fn strict_recovery(&self) -> bool {
matches!(self.recovery_mode, RecoveryMode::Strict)
}
/// Returns a reference to the checkpoint encryption key, dereferencing through
/// the Arc<Zeroizing<>> wrapper for use with checkpoint functions.
pub fn checkpoint_key(&self) -> Option<&[u8; 32]> {
self.checkpoint_encryption_key.as_ref().map(|arc| &***arc)
}
/// Returns a reference to the manifest HMAC key, dereferencing through
/// the Arc<Zeroizing<>> wrapper for use with manifest functions.
pub fn hmac_key(&self) -> Option<&[u8]> {
self.manifest_hmac_key.as_ref().map(|arc| &***arc as &[u8])
}
/// Returns a reference to the cursor signing HMAC key, dereferencing
/// through the Arc<Zeroizing<>> wrapper. Returns `None` when cursor
/// signing is disabled (legacy back-compat).
pub fn cursor_signing_key(&self) -> Option<&[u8; 32]> {
self.cursor_signing_key.as_ref().map(|arc| &***arc)
}
/// Sets the cursor signing HMAC key, wrapping it in Arc<Zeroizing<>> for
/// secure memory handling.
pub fn with_cursor_signing_key(mut self, key: [u8; 32]) -> Self {
self.cursor_signing_key = Some(Arc::new(Zeroizing::new(key)));
self
}
/// Sets the checkpoint encryption key, wrapping it in Arc<Zeroizing<>> for
/// secure memory handling.
pub fn with_checkpoint_key(mut self, key: [u8; 32]) -> Self {
self.checkpoint_encryption_key = Some(Arc::new(Zeroizing::new(key)));
self
}
/// Sets the manifest HMAC key, wrapping it in Arc<Zeroizing<>> for
/// secure memory handling.
pub fn with_hmac_key(mut self, key: Vec<u8>) -> Self {
self.manifest_hmac_key = Some(Arc::new(Zeroizing::new(key)));
self
}
/// Sets the zstd compression level used for checkpoints.
pub fn with_checkpoint_compression_level(mut self, level: i32) -> Self {
self.checkpoint_compression_level = level;
self
}
/// Sets how many recent checkpoints to retain in the manifest. Values below
/// 1 are clamped to 1 by [`AedbConfig::checkpoint_retention_count`].
pub fn with_checkpoint_retention_count(mut self, count: usize) -> Self {
self.checkpoint_retention_count = count;
self
}
/// Effective checkpoint retention count, clamped to at least 1 so the live
/// checkpoint is always kept.
pub fn checkpoint_retention_count(&self) -> usize {
self.checkpoint_retention_count.max(1)
}
}