auradb 0.1.0

High-performance Rust storage engine with WAL-time KV separation, RL-driven compaction, and learned indexes
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
use serde::{Deserialize, Serialize};
use std::path::PathBuf;

/// Configuration for the AuraDB storage engine
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Config {
    /// Database directory path
    pub db_path: PathBuf,
    
    /// WAL configuration
    pub wal: WalConfig,
    
    /// Value log configuration
    pub value_log: ValueLogConfig,
    
    /// Memtable configuration
    pub memtable: MemtableConfig,
    
    /// SST configuration
    pub sst: SstConfig,
    
    /// Compaction configuration
    pub compaction: CompactionConfig,
    
    /// Cache configuration
    pub cache: CacheConfig,
    
    /// Learned index configuration
    pub learned_index: LearnedIndexConfig,
    
    /// RL agent configuration
    pub rl_agent: RlAgentConfig,
    
    /// Performance tuning
    pub performance: PerformanceConfig,
}

impl Default for Config {
    fn default() -> Self {
        Self {
            db_path: PathBuf::from("./auradb_data"),
            wal: WalConfig::default(),
            value_log: ValueLogConfig::default(),
            memtable: MemtableConfig::default(),
            sst: SstConfig::default(),
            compaction: CompactionConfig::default(),
            cache: CacheConfig::default(),
            learned_index: LearnedIndexConfig::default(),
            rl_agent: RlAgentConfig::default(),
            performance: PerformanceConfig::default(),
        }
    }
}

/// WAL (Write-Ahead Log) configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WalConfig {
    /// WAL directory path
    pub wal_path: PathBuf,
    /// Maximum WAL file size in bytes
    pub max_file_size: u64,
    /// Whether to use async WAL writes
    pub async_writes: bool,
    /// WAL sync policy
    pub sync_policy: WalSyncPolicy,
    /// WAL buffer size in bytes
    pub buffer_size: usize,
}

impl Default for WalConfig {
    fn default() -> Self {
        Self {
            wal_path: PathBuf::from("./auradb_data/wal"),
            max_file_size: 64 * 1024 * 1024, // 64MB
            async_writes: true,
            sync_policy: WalSyncPolicy::EveryWrite,
            buffer_size: 64 * 1024, // 64KB
        }
    }
}

/// WAL sync policy
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum WalSyncPolicy {
    /// Sync every write (safest, slowest)
    EveryWrite,
    /// Sync every N writes
    EveryNWrites(u64),
    /// Sync every N milliseconds
    EveryNMs(u64),
    /// Manual sync only
    Manual,
}

/// Value log configuration for WAL-time KV separation
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValueLogConfig {
    /// Value log directory path
    pub vlog_path: PathBuf,
    /// Maximum segment size in bytes
    pub max_segment_size: u64,
    /// Value size threshold for separation (bytes)
    pub separation_threshold: usize,
    /// Number of parallel write queues
    pub write_queues: usize,
    /// Value log cache size in bytes
    pub cache_size: usize,
    /// Whether to compress values
    pub compress_values: bool,
    /// Compression algorithm
    pub compression_algorithm: CompressionAlgorithm,
}

impl Default for ValueLogConfig {
    fn default() -> Self {
        Self {
            vlog_path: PathBuf::from("./auradb_data/vlog"),
            max_segment_size: 256 * 1024 * 1024, // 256MB
            separation_threshold: 1024, // 1KB
            write_queues: 4,
            cache_size: 64 * 1024 * 1024, // 64MB
            compress_values: true,
            compression_algorithm: CompressionAlgorithm::Lz4,
        }
    }
}

/// Compression algorithms
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Copy)]
pub enum CompressionAlgorithm {
    /// No compression
    None,
    /// Lz4 compression (fast)
    Lz4,
    /// Zstandard compression (good compression ratio)
    Zstd,
    /// Snappy compression (balanced)
    Snappy,
}

/// Memtable configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemtableConfig {
    /// Maximum memtable size in bytes
    pub max_size: usize,
    /// Memtable implementation
    pub implementation: MemtableImpl,
    /// Number of memtables
    pub count: usize,
    /// Flush threshold (percentage of max_size)
    pub flush_threshold: f64,
}

impl Default for MemtableConfig {
    fn default() -> Self {
        Self {
            max_size: 64 * 1024 * 1024, // 64MB
            implementation: MemtableImpl::SkipList,
            count: 2,
            flush_threshold: 0.8, // 80%
        }
    }
}

/// Memtable implementation type
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum MemtableImpl {
    /// Skip list implementation
    SkipList,
    /// Adaptive Radix Tree (ART)
    Art,
    /// B-tree implementation
    BTree,
}

/// SST (Sorted String Table) configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SstConfig {
    /// SST directory path
    pub sst_path: PathBuf,
    /// Target file size in bytes
    pub target_file_size: u64,
    /// Block size in bytes
    pub block_size: usize,
    /// Whether to use Bloom filters
    pub use_bloom_filters: bool,
    /// Bloom filter bits per key
    pub bloom_bits_per_key: f64,
    /// Whether to use Ribbon filters
    pub use_ribbon_filters: bool,
    /// Compression algorithm for SST blocks
    pub compression: CompressionAlgorithm,
}

impl Default for SstConfig {
    fn default() -> Self {
        Self {
            sst_path: PathBuf::from("./auradb_data/sst"),
            target_file_size: 64 * 1024 * 1024, // 64MB
            block_size: 64 * 1024, // 64KB
            use_bloom_filters: true,
            bloom_bits_per_key: 10.0,
            use_ribbon_filters: false,
            compression: CompressionAlgorithm::Lz4,
        }
    }
}

/// Compaction configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CompactionConfig {
    /// Compaction strategy
    pub strategy: CompactionStrategy,
    /// Maximum number of background threads
    pub max_threads: usize,
    /// I/O rate limit in MB/s
    pub io_rate_limit: Option<u64>,
    /// Whether to use RL-driven compaction
    pub use_rl_agent: bool,
    /// Compaction trigger thresholds
    pub triggers: CompactionTriggers,
}

impl Default for CompactionConfig {
    fn default() -> Self {
        Self {
            strategy: CompactionStrategy::Leveled,
            max_threads: 4,
            io_rate_limit: Some(100), // 100 MB/s
            use_rl_agent: true,
            triggers: CompactionTriggers::default(),
        }
    }
}

/// Compaction strategy
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum CompactionStrategy {
    /// Leveled compaction (RocksDB-style)
    Leveled,
    /// Tiered compaction
    Tiered,
    /// Flexible LSM (FLSM) - can switch between strategies
    Flexible,
}

/// Compaction trigger thresholds
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CompactionTriggers {
    /// Level 0 file count threshold
    pub level0_files: usize,
    /// Level size ratio threshold
    pub level_size_ratio: f64,
    /// Write amplification threshold
    pub write_amplification: f64,
}

impl Default for CompactionTriggers {
    fn default() -> Self {
        Self {
            level0_files: 4,
            level_size_ratio: 10.0,
            write_amplification: 5.0,
        }
    }
}

/// Cache configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CacheConfig {
    /// Block cache size in bytes
    pub block_cache_size: usize,
    /// Value log cache size in bytes
    pub vlog_cache_size: usize,
    /// Cache eviction policy
    pub eviction_policy: EvictionPolicy,
    /// Whether to use unified cache
    pub unified_cache: bool,
}

impl Default for CacheConfig {
    fn default() -> Self {
        Self {
            block_cache_size: 256 * 1024 * 1024, // 256MB
            vlog_cache_size: 64 * 1024 * 1024, // 64MB
            eviction_policy: EvictionPolicy::Arc,
            unified_cache: true,
        }
    }
}

/// Cache eviction policy
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum EvictionPolicy {
    /// LRU (Least Recently Used)
    Lru,
    /// ARC (Adaptive Replacement Cache)
    Arc,
    /// TinyLFU
    TinyLfu,
}

/// Learned index configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LearnedIndexConfig {
    /// Whether to enable learned indexes
    pub enabled: bool,
    /// Model type to use
    pub model_type: ModelType,
    /// Training frequency (every N operations)
    pub training_frequency: usize,
    /// Whether to use online tuning
    pub online_tuning: bool,
    /// Fallback search method
    pub fallback_method: FallbackMethod,
}

impl Default for LearnedIndexConfig {
    fn default() -> Self {
        Self {
            enabled: true,
            model_type: ModelType::PiecewiseLinear,
            training_frequency: 10000,
            online_tuning: true,
            fallback_method: FallbackMethod::BinarySearch,
        }
    }
}

/// Learned index model type
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ModelType {
    /// Piecewise linear regression
    PiecewiseLinear,
    /// Recursive Model Index (RMI)
    Rmi,
    /// Tiny neural network
    TinyNn,
}

/// Fallback search method when learned index fails
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum FallbackMethod {
    /// Binary search
    BinarySearch,
    /// Fence pointers
    FencePointers,
    /// Bloom filter + scan
    BloomScan,
}

/// RL agent configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RlAgentConfig {
    /// Whether to enable RL agent
    pub enabled: bool,
    /// Learning rate
    pub learning_rate: f64,
    /// Exploration rate (epsilon)
    pub exploration_rate: f64,
    /// State update frequency
    pub state_update_frequency: usize,
    /// Whether to use offline training
    pub offline_training: bool,
    /// Training data path
    pub training_data_path: Option<PathBuf>,
}

impl Default for RlAgentConfig {
    fn default() -> Self {
        Self {
            enabled: true,
            learning_rate: 0.01,
            exploration_rate: 0.1,
            state_update_frequency: 1000,
            offline_training: false,
            training_data_path: None,
        }
    }
}

/// Performance tuning configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceConfig {
    /// Number of worker threads
    pub worker_threads: usize,
    /// I/O buffer size
    pub io_buffer_size: usize,
    /// Whether to use direct I/O
    pub direct_io: bool,
    /// Whether to use memory-mapped files
    pub memory_mapped: bool,
    /// NUMA awareness
    pub numa_aware: bool,
}

impl Default for PerformanceConfig {
    fn default() -> Self {
        Self {
            worker_threads: num_cpus::get(),
            io_buffer_size: 1024 * 1024, // 1MB
            direct_io: false,
            memory_mapped: true,
            numa_aware: false,
        }
    }
}

impl Config {
    /// Create a new configuration with default values
    pub fn new() -> Self {
        Self::default()
    }

    /// Set the database path
    pub fn with_db_path(mut self, path: PathBuf) -> Self {
        self.db_path = path;
        self
    }

    /// Set WAL configuration
    pub fn with_wal(mut self, wal: WalConfig) -> Self {
        self.wal = wal;
        self
    }

    /// Set value log configuration
    pub fn with_value_log(mut self, vlog: ValueLogConfig) -> Self {
        self.value_log = vlog;
        self
    }

    /// Set memtable configuration
    pub fn with_memtable(mut self, memtable: MemtableConfig) -> Self {
        self.memtable = memtable;
        self
    }

    /// Set SST configuration
    pub fn with_sst(mut self, sst: SstConfig) -> Self {
        self.sst = sst;
        self
    }

    /// Set compaction configuration
    pub fn with_compaction(mut self, compaction: CompactionConfig) -> Self {
        self.compaction = compaction;
        self
    }

    /// Set cache configuration
    pub fn with_cache(mut self, cache: CacheConfig) -> Self {
        self.cache = cache;
        self
    }

    /// Set learned index configuration
    pub fn with_learned_index(mut self, learned_index: LearnedIndexConfig) -> Self {
        self.learned_index = learned_index;
        self
    }

    /// Set RL agent configuration
    pub fn with_rl_agent(mut self, rl_agent: RlAgentConfig) -> Self {
        self.rl_agent = rl_agent;
        self
    }

    /// Set performance configuration
    pub fn with_performance(mut self, performance: PerformanceConfig) -> Self {
        self.performance = performance;
        self
    }

    /// Validate the configuration
    pub fn validate(&self) -> Result<(), String> {
        if self.wal.max_file_size == 0 {
            return Err("WAL max file size must be greater than 0".to_string());
        }
        if self.value_log.max_segment_size == 0 {
            return Err("Value log max segment size must be greater than 0".to_string());
        }
        if self.memtable.max_size == 0 {
            return Err("Memtable max size must be greater than 0".to_string());
        }
        if self.sst.target_file_size == 0 {
            return Err("SST target file size must be greater than 0".to_string());
        }
        if self.cache.block_cache_size == 0 {
            return Err("Block cache size must be greater than 0".to_string());
        }
        Ok(())
    }
}