velesdb_core/
config.rs

1//! `VelesDB` Configuration Module
2//!
3//! Provides configuration file support via `velesdb.toml`, environment variables,
4//! and runtime overrides.
5//!
6//! # Priority (highest to lowest)
7//!
8//! 1. Runtime overrides (API, REPL)
9//! 2. Environment variables (`VELESDB_*`)
10//! 3. Configuration file (`velesdb.toml`)
11//! 4. Default values
12
13use figment::{
14    providers::{Env, Format, Serialized, Toml},
15    Figment,
16};
17use serde::{Deserialize, Serialize};
18use std::path::Path;
19use thiserror::Error;
20
21// ---------------------------------------------------------------------------
22// QuantizationType enum (PQ-06)
23// ---------------------------------------------------------------------------
24
25/// Default value for PQ codebook size (`k`).
26const fn default_k() -> usize {
27    256
28}
29
30/// Default value for PQ oversampling factor.
31#[allow(clippy::unnecessary_wraps)]
32const fn default_oversampling() -> Option<u32> {
33    Some(4)
34}
35
36/// Quantization type for a collection (PQ-06).
37///
38/// Determines which quantization algorithm is applied to stored vectors.
39/// Uses a serde-tagged representation for the new format, with backward
40/// compatibility via [`QuantizationConfig`]'s custom deserializer.
41#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
42#[serde(tag = "type", rename_all = "lowercase")]
43pub enum QuantizationType {
44    /// No quantization -- full-precision vectors.
45    #[default]
46    None,
47    /// Scalar quantization to 8-bit integers (4x compression).
48    #[serde(alias = "sq8")]
49    SQ8,
50    /// Binary quantization (32x compression).
51    Binary,
52    /// Product quantization with configurable subspaces.
53    #[serde(alias = "pq")]
54    PQ {
55        /// Number of subspaces (dimension must be divisible by `m`).
56        m: usize,
57        /// Codebook size per subspace.
58        #[serde(default = "default_k")]
59        k: usize,
60        /// Enable Optimized Product Quantization (OPQ) rotation.
61        #[serde(default)]
62        opq_enabled: bool,
63        /// Oversampling factor for training. `None` disables oversampling.
64        #[serde(default = "default_oversampling")]
65        oversampling: Option<u32>,
66    },
67    /// Randomized Binary Quantization.
68    #[serde(alias = "rabitq")]
69    RaBitQ,
70}
71
72impl QuantizationType {
73    /// Returns `true` if this is Product Quantization.
74    #[must_use]
75    pub const fn is_pq(&self) -> bool {
76        matches!(self, Self::PQ { .. })
77    }
78
79    /// Returns `true` if this is Randomized Binary Quantization.
80    #[must_use]
81    pub const fn is_rabitq(&self) -> bool {
82        matches!(self, Self::RaBitQ)
83    }
84}
85
86/// Configuration errors.
87#[derive(Error, Debug)]
88pub enum ConfigError {
89    /// Failed to parse configuration file.
90    #[error("Failed to parse configuration: {0}")]
91    ParseError(String),
92
93    /// Invalid configuration value.
94    #[error("Invalid configuration value for '{key}': {message}")]
95    InvalidValue {
96        /// Configuration key that failed validation.
97        key: String,
98        /// Validation error message.
99        message: String,
100    },
101
102    /// Configuration file not found.
103    #[error("Configuration file not found: {0}")]
104    FileNotFound(String),
105
106    /// IO error.
107    #[error("IO error: {0}")]
108    IoError(#[from] std::io::Error),
109}
110
111/// Search mode presets.
112#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
113#[serde(rename_all = "snake_case")]
114pub enum SearchMode {
115    /// Fast search with `ef_search=64`, ~92% recall.
116    Fast,
117    /// Balanced search with `ef_search=128`, ~99% recall (default).
118    #[default]
119    Balanced,
120    /// Accurate search with `ef_search=512`, ~100% recall.
121    Accurate,
122    /// Perfect recall with bruteforce, 100% guaranteed.
123    Perfect,
124}
125
126impl SearchMode {
127    /// Returns the `ef_search` value for this mode.
128    #[must_use]
129    pub fn ef_search(&self) -> usize {
130        match self {
131            Self::Fast => 64,
132            Self::Balanced => 128,
133            Self::Accurate => 512,
134            Self::Perfect => usize::MAX, // Signals bruteforce
135        }
136    }
137}
138
139/// Search configuration section.
140#[derive(Debug, Clone, Serialize, Deserialize)]
141#[serde(default)]
142pub struct SearchConfig {
143    /// Default search mode.
144    pub default_mode: SearchMode,
145    /// Override `ef_search` (if set, overrides mode).
146    pub ef_search: Option<usize>,
147    /// Maximum results per query.
148    pub max_results: usize,
149    /// Query timeout in milliseconds.
150    pub query_timeout_ms: u64,
151}
152
153impl Default for SearchConfig {
154    fn default() -> Self {
155        Self {
156            default_mode: SearchMode::Balanced,
157            ef_search: None,
158            max_results: 1000,
159            query_timeout_ms: 30000,
160        }
161    }
162}
163
164/// HNSW index configuration section.
165#[derive(Debug, Clone, Default, Serialize, Deserialize)]
166#[serde(default)]
167pub struct HnswConfig {
168    /// Number of connections per node (M parameter).
169    /// `None` = auto based on dimension.
170    pub m: Option<usize>,
171    /// Size of the candidate pool during construction.
172    /// `None` = auto based on dimension.
173    pub ef_construction: Option<usize>,
174    /// Maximum number of layers (0 = auto).
175    pub max_layers: usize,
176}
177
178/// Server-layer configuration types (HTTP transport, logging, storage paths).
179///
180/// These types are intentionally separated from the core engine configuration
181/// (`SearchConfig`, `HnswConfig`, `LimitsConfig`) to enforce layer boundaries.
182/// Import via `config::server::ServerConfig` or use the crate-root re-exports.
183pub mod server {
184    use serde::{Deserialize, Serialize};
185
186    /// Storage configuration section.
187    #[derive(Debug, Clone, Serialize, Deserialize)]
188    #[serde(default)]
189    pub struct StorageConfig {
190        /// Data directory path.
191        pub data_dir: String,
192        /// Storage mode: `"mmap"` or `"memory"`.
193        pub storage_mode: String,
194        /// Mmap cache size in megabytes.
195        pub mmap_cache_mb: usize,
196        /// Vector alignment in bytes.
197        pub vector_alignment: usize,
198    }
199
200    impl Default for StorageConfig {
201        fn default() -> Self {
202            Self {
203                data_dir: "./velesdb_data".to_string(),
204                storage_mode: "mmap".to_string(),
205                mmap_cache_mb: 1024,
206                vector_alignment: 64,
207            }
208        }
209    }
210
211    /// Server configuration section.
212    #[derive(Debug, Clone, Serialize, Deserialize)]
213    #[serde(default)]
214    pub struct ServerConfig {
215        /// Host address.
216        pub host: String,
217        /// Port number.
218        pub port: u16,
219        /// Number of worker threads (0 = auto).
220        pub workers: usize,
221        /// Maximum HTTP body size in bytes.
222        pub max_body_size: usize,
223        /// Enable CORS.
224        pub cors_enabled: bool,
225        /// CORS allowed origins.
226        pub cors_origins: Vec<String>,
227    }
228
229    impl Default for ServerConfig {
230        fn default() -> Self {
231            Self {
232                host: "127.0.0.1".to_string(),
233                port: 8080,
234                workers: 0,
235                max_body_size: 104_857_600,
236                cors_enabled: false,
237                cors_origins: vec!["*".to_string()],
238            }
239        }
240    }
241
242    /// Logging configuration section.
243    #[derive(Debug, Clone, Serialize, Deserialize)]
244    #[serde(default)]
245    pub struct LoggingConfig {
246        /// Log level: `error`, `warn`, `info`, `debug`, `trace`.
247        pub level: String,
248        /// Log format: `text` or `json`.
249        pub format: String,
250        /// Log file path (empty = stdout).
251        pub file: String,
252    }
253
254    impl Default for LoggingConfig {
255        fn default() -> Self {
256            Self {
257                level: "info".to_string(),
258                format: "text".to_string(),
259                file: String::new(),
260            }
261        }
262    }
263}
264
265// Backward-compatible re-exports at module level.
266pub use server::{LoggingConfig, ServerConfig, StorageConfig};
267
268/// Limits configuration section.
269#[derive(Debug, Clone, Serialize, Deserialize)]
270#[serde(default)]
271pub struct LimitsConfig {
272    /// Maximum vector dimensions.
273    pub max_dimensions: usize,
274    /// Maximum vectors per collection.
275    pub max_vectors_per_collection: usize,
276    /// Maximum number of collections.
277    pub max_collections: usize,
278    /// Maximum payload size in bytes.
279    pub max_payload_size: usize,
280    /// Maximum vectors for perfect mode (bruteforce).
281    pub max_perfect_mode_vectors: usize,
282}
283
284impl Default for LimitsConfig {
285    fn default() -> Self {
286        Self {
287            max_dimensions: 4096,
288            max_vectors_per_collection: 100_000_000,
289            max_collections: 1000,
290            max_payload_size: 1_048_576, // 1 MB
291            max_perfect_mode_vectors: 500_000,
292        }
293    }
294}
295
296/// Quantization configuration section (EPIC-073/US-005, PQ-06).
297///
298/// Supports two JSON shapes for backward compatibility:
299/// - **Old format:** `{"default_type": "sq8", "rerank_enabled": true, ...}`
300/// - **New format:** `{"mode": {"type": "pq", "m": 8, ...}, "rerank_enabled": true, ...}`
301#[derive(Debug, Clone, Serialize)]
302pub struct QuantizationConfig {
303    /// Quantization mode (replaces the old `default_type` string).
304    pub mode: QuantizationType,
305    /// Enable reranking after quantized search.
306    pub rerank_enabled: bool,
307    /// Reranking multiplier for candidates.
308    pub rerank_multiplier: usize,
309    /// Auto-enable quantization for large collections (EPIC-073/US-005).
310    pub auto_quantization: bool,
311    /// Threshold for auto-quantization (number of vectors).
312    pub auto_quantization_threshold: usize,
313}
314
315impl Default for QuantizationConfig {
316    fn default() -> Self {
317        Self {
318            mode: QuantizationType::None,
319            rerank_enabled: true,
320            rerank_multiplier: 2,
321            auto_quantization: true,
322            auto_quantization_threshold: 10_000,
323        }
324    }
325}
326
327impl QuantizationConfig {
328    /// Returns a reference to the quantization mode.
329    #[must_use]
330    pub const fn mode(&self) -> &QuantizationType {
331        &self.mode
332    }
333
334    /// Returns whether quantization should be used based on vector count (EPIC-073/US-005).
335    #[must_use]
336    pub fn should_quantize(&self, vector_count: usize) -> bool {
337        self.auto_quantization && vector_count >= self.auto_quantization_threshold
338    }
339}
340
341// ---------------------------------------------------------------------------
342// Custom Deserialize for backward compatibility (PQ-06)
343// ---------------------------------------------------------------------------
344
345impl<'de> Deserialize<'de> for QuantizationConfig {
346    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
347    where
348        D: serde::Deserializer<'de>,
349    {
350        /// Raw intermediate struct that accepts either old or new format.
351        #[derive(Deserialize)]
352        struct RawQuantizationConfig {
353            /// New format: structured mode object.
354            #[serde(default)]
355            mode: Option<QuantizationType>,
356            /// Old format: plain string ("none", "sq8", "binary").
357            #[serde(default)]
358            default_type: Option<String>,
359            #[serde(default = "default_rerank_enabled")]
360            rerank_enabled: bool,
361            #[serde(default = "default_rerank_multiplier")]
362            rerank_multiplier: usize,
363            #[serde(default = "default_auto_quantization")]
364            auto_quantization: bool,
365            #[serde(default = "default_auto_quantization_threshold")]
366            auto_quantization_threshold: usize,
367        }
368
369        fn default_rerank_enabled() -> bool {
370            true
371        }
372        fn default_rerank_multiplier() -> usize {
373            2
374        }
375        fn default_auto_quantization() -> bool {
376            true
377        }
378        fn default_auto_quantization_threshold() -> usize {
379            10_000
380        }
381
382        let raw = RawQuantizationConfig::deserialize(deserializer)?;
383
384        let mode = if let Some(m) = raw.mode {
385            m
386        } else if let Some(ref s) = raw.default_type {
387            match s.as_str() {
388                "none" | "" => QuantizationType::None,
389                "sq8" => QuantizationType::SQ8,
390                "binary" => QuantizationType::Binary,
391                other => {
392                    return Err(serde::de::Error::custom(format!(
393                        "unknown quantization type: '{other}'"
394                    )));
395                }
396            }
397        } else {
398            QuantizationType::None
399        };
400
401        Ok(Self {
402            mode,
403            rerank_enabled: raw.rerank_enabled,
404            rerank_multiplier: raw.rerank_multiplier,
405            auto_quantization: raw.auto_quantization,
406            auto_quantization_threshold: raw.auto_quantization_threshold,
407        })
408    }
409}
410
411// ---------------------------------------------------------------------------
412// WAL batch commit configuration
413// ---------------------------------------------------------------------------
414
415/// Default commit delay in microseconds for WAL group commit.
416const fn default_commit_delay_us() -> u64 {
417    100
418}
419
420/// Default maximum entries per WAL batch.
421const fn default_max_batch_size() -> usize {
422    128
423}
424
425/// Configuration for WAL group commit batching.
426///
427/// When enabled, multiple concurrent writes are batched into a single
428/// `sync_all()` call, amortizing the fsync cost across the batch.
429///
430/// # Example (TOML)
431///
432/// ```toml
433/// [wal_batch]
434/// enabled = true
435/// commit_delay_us = 200
436/// max_batch_size = 256
437/// ```
438#[derive(Debug, Clone, Serialize, Deserialize)]
439pub struct WalBatchConfig {
440    /// Whether group commit is enabled. Default: `false`.
441    #[serde(default)]
442    pub enabled: bool,
443    /// Maximum delay in microseconds before flushing a batch. Default: `100`.
444    #[serde(default = "default_commit_delay_us")]
445    pub commit_delay_us: u64,
446    /// Maximum number of entries per batch. Default: `128`.
447    #[serde(default = "default_max_batch_size")]
448    pub max_batch_size: usize,
449}
450
451impl Default for WalBatchConfig {
452    fn default() -> Self {
453        Self {
454            enabled: false,
455            commit_delay_us: 100,
456            max_batch_size: 128,
457        }
458    }
459}
460
461/// Main `VelesDB` configuration structure.
462#[derive(Debug, Clone, Serialize, Deserialize, Default)]
463#[serde(default)]
464pub struct VelesConfig {
465    /// Search configuration.
466    pub search: SearchConfig,
467    /// HNSW index configuration.
468    pub hnsw: HnswConfig,
469    /// Storage configuration.
470    pub storage: StorageConfig,
471    /// Limits configuration.
472    pub limits: LimitsConfig,
473    /// Server configuration.
474    pub server: ServerConfig,
475    /// Logging configuration.
476    pub logging: LoggingConfig,
477    /// Quantization configuration.
478    pub quantization: QuantizationConfig,
479    /// WAL group commit batching configuration.
480    pub wal_batch: WalBatchConfig,
481}
482
483impl VelesConfig {
484    /// Loads configuration from default sources.
485    ///
486    /// Priority: defaults < file < environment variables.
487    ///
488    /// # Errors
489    ///
490    /// Returns an error if configuration parsing fails.
491    pub fn load() -> Result<Self, ConfigError> {
492        Self::load_from_path("velesdb.toml")
493    }
494
495    /// Loads configuration from a specific file path.
496    ///
497    /// # Arguments
498    ///
499    /// * `path` - Path to the configuration file.
500    ///
501    /// # Errors
502    ///
503    /// Returns an error if configuration parsing fails.
504    pub fn load_from_path<P: AsRef<Path>>(path: P) -> Result<Self, ConfigError> {
505        let figment = Figment::new()
506            .merge(Serialized::defaults(Self::default()))
507            .merge(Toml::file(path.as_ref()))
508            .merge(Env::prefixed("VELESDB_").split("_").lowercase(false));
509
510        figment
511            .extract()
512            .map_err(|e| ConfigError::ParseError(e.to_string()))
513    }
514
515    /// Creates a configuration from a TOML string.
516    ///
517    /// # Arguments
518    ///
519    /// * `toml_str` - TOML configuration string.
520    ///
521    /// # Errors
522    ///
523    /// Returns an error if parsing fails.
524    pub fn from_toml(toml_str: &str) -> Result<Self, ConfigError> {
525        let figment = Figment::new()
526            .merge(Serialized::defaults(Self::default()))
527            .merge(Toml::string(toml_str));
528
529        figment
530            .extract()
531            .map_err(|e| ConfigError::ParseError(e.to_string()))
532    }
533
534    /// Validates the configuration.
535    ///
536    /// # Errors
537    ///
538    /// Returns an error if any configuration value is invalid.
539    pub fn validate(&self) -> Result<(), ConfigError> {
540        // Validate search config
541        if let Some(ef) = self.search.ef_search {
542            if !(16..=4096).contains(&ef) {
543                return Err(ConfigError::InvalidValue {
544                    key: "search.ef_search".to_string(),
545                    message: format!("value {ef} is out of range [16, 4096]"),
546                });
547            }
548        }
549
550        if self.search.max_results == 0 || self.search.max_results > 10000 {
551            return Err(ConfigError::InvalidValue {
552                key: "search.max_results".to_string(),
553                message: format!(
554                    "value {} is out of range [1, 10000]",
555                    self.search.max_results
556                ),
557            });
558        }
559
560        // Validate HNSW config
561        if let Some(m) = self.hnsw.m {
562            if !(4..=128).contains(&m) {
563                return Err(ConfigError::InvalidValue {
564                    key: "hnsw.m".to_string(),
565                    message: format!("value {m} is out of range [4, 128]"),
566                });
567            }
568        }
569
570        if let Some(ef) = self.hnsw.ef_construction {
571            if !(100..=2000).contains(&ef) {
572                return Err(ConfigError::InvalidValue {
573                    key: "hnsw.ef_construction".to_string(),
574                    message: format!("value {ef} is out of range [100, 2000]"),
575                });
576            }
577        }
578
579        // Validate limits
580        if self.limits.max_dimensions == 0 || self.limits.max_dimensions > 65536 {
581            return Err(ConfigError::InvalidValue {
582                key: "limits.max_dimensions".to_string(),
583                message: format!(
584                    "value {} is out of range [1, 65536]",
585                    self.limits.max_dimensions
586                ),
587            });
588        }
589
590        // Validate server config
591        if self.server.port < 1024 {
592            return Err(ConfigError::InvalidValue {
593                key: "server.port".to_string(),
594                message: format!("value {} must be >= 1024", self.server.port),
595            });
596        }
597
598        // Validate storage mode
599        let valid_modes = ["mmap", "memory"];
600        if !valid_modes.contains(&self.storage.storage_mode.as_str()) {
601            return Err(ConfigError::InvalidValue {
602                key: "storage.storage_mode".to_string(),
603                message: format!(
604                    "value '{}' is invalid, expected one of: {:?}",
605                    self.storage.storage_mode, valid_modes
606                ),
607            });
608        }
609
610        // Validate logging level
611        let valid_levels = ["error", "warn", "info", "debug", "trace"];
612        if !valid_levels.contains(&self.logging.level.as_str()) {
613            return Err(ConfigError::InvalidValue {
614                key: "logging.level".to_string(),
615                message: format!(
616                    "value '{}' is invalid, expected one of: {:?}",
617                    self.logging.level, valid_levels
618                ),
619            });
620        }
621
622        Ok(())
623    }
624
625    /// Returns the effective `ef_search` value.
626    ///
627    /// Uses explicit `ef_search` if set, otherwise derives from search mode.
628    #[must_use]
629    pub fn effective_ef_search(&self) -> usize {
630        self.search
631            .ef_search
632            .unwrap_or_else(|| self.search.default_mode.ef_search())
633    }
634
635    /// Serializes the configuration to TOML.
636    ///
637    /// # Errors
638    ///
639    /// Returns an error if serialization fails.
640    pub fn to_toml(&self) -> Result<String, ConfigError> {
641        toml::to_string_pretty(self).map_err(|e| ConfigError::ParseError(e.to_string()))
642    }
643}
velesdb_core/config.rs

velesdb_core/
config.rs
