oxirs_vec/real_time_embedding_pipeline/
config.rs

1//! Configuration types for the real-time embedding pipeline
2
3use serde::{Deserialize, Serialize};
4
5/// Pipeline configuration for real-time embedding updates
6#[derive(Debug, Clone, Serialize, Deserialize)]
7pub struct PipelineConfig {
8    /// Maximum batch size for processing
9    pub max_batch_size: usize,
10    /// Batch timeout in milliseconds
11    pub batch_timeout_ms: u64,
12    /// Maximum concurrent updates
13    pub max_concurrent_updates: usize,
14    /// Buffer size for each stream
15    pub stream_buffer_size: usize,
16    /// Update timeout in seconds
17    pub update_timeout_seconds: u64,
18    /// Consistency level
19    pub consistency_level: ConsistencyLevel,
20    /// Backpressure strategy
21    pub backpressure_strategy: BackpressureStrategy,
22    /// Retry configuration
23    pub retry_config: RetryConfig,
24    /// Performance monitoring settings
25    pub monitoring_config: MonitoringConfig,
26    /// Version control settings
27    pub version_control: VersionControlConfig,
28    /// Quality assurance settings
29    pub quality_assurance: QualityAssuranceConfig,
30}
31
32impl Default for PipelineConfig {
33    fn default() -> Self {
34        Self {
35            max_batch_size: 1000,
36            batch_timeout_ms: 100,
37            max_concurrent_updates: 10,
38            stream_buffer_size: 10000,
39            update_timeout_seconds: 30,
40            consistency_level: ConsistencyLevel::Session,
41            backpressure_strategy: BackpressureStrategy::Adaptive,
42            retry_config: RetryConfig::default(),
43            monitoring_config: MonitoringConfig::default(),
44            version_control: VersionControlConfig::default(),
45            quality_assurance: QualityAssuranceConfig::default(),
46        }
47    }
48}
49
50/// Consistency levels for updates
51#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
52pub enum ConsistencyLevel {
53    /// Eventually consistent - fast but may have temporary inconsistencies
54    Eventual,
55    /// Session consistent - consistent within a session
56    Session,
57    /// Strong consistency - always consistent but slower
58    Strong,
59    /// Causal consistency - maintains causal ordering
60    Causal,
61    /// Monotonic read consistency
62    MonotonicRead,
63}
64
65/// Backpressure handling strategies
66#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
67pub enum BackpressureStrategy {
68    /// Drop oldest updates when buffer is full
69    DropOldest,
70    /// Drop newest updates when buffer is full
71    DropNewest,
72    /// Block until buffer has space
73    Block,
74    /// Adaptive strategy based on system load
75    Adaptive,
76    /// Exponential backoff
77    ExponentialBackoff {
78        initial_delay_ms: u64,
79        max_delay_ms: u64,
80    },
81}
82
83/// Retry configuration
84#[derive(Debug, Clone, Serialize, Deserialize)]
85pub struct RetryConfig {
86    /// Maximum number of retries
87    pub max_retries: usize,
88    /// Base delay between retries
89    pub base_delay_ms: u64,
90    /// Maximum delay between retries
91    pub max_delay_ms: u64,
92    /// Exponential backoff multiplier
93    pub backoff_multiplier: f64,
94    /// Jitter factor for randomization
95    pub jitter_factor: f64,
96}
97
98impl Default for RetryConfig {
99    fn default() -> Self {
100        Self {
101            max_retries: 3,
102            base_delay_ms: 100,
103            max_delay_ms: 5000,
104            backoff_multiplier: 2.0,
105            jitter_factor: 0.1,
106        }
107    }
108}
109
110/// Monitoring configuration
111#[derive(Debug, Clone, Serialize, Deserialize)]
112pub struct MonitoringConfig {
113    /// Metrics collection interval
114    pub metrics_interval_ms: u64,
115    /// Performance alert thresholds
116    pub alert_thresholds: AlertThresholds,
117    /// Enable detailed tracing
118    pub enable_tracing: bool,
119    /// Metrics retention period
120    pub metrics_retention_hours: u64,
121}
122
123impl Default for MonitoringConfig {
124    fn default() -> Self {
125        Self {
126            metrics_interval_ms: 1000,
127            alert_thresholds: AlertThresholds::default(),
128            enable_tracing: false,
129            metrics_retention_hours: 24,
130        }
131    }
132}
133
134/// Alert thresholds for monitoring
135#[derive(Debug, Clone, Serialize, Deserialize)]
136pub struct AlertThresholds {
137    /// Maximum processing latency in milliseconds
138    pub max_processing_latency_ms: u64,
139    /// Maximum queue depth
140    pub max_queue_depth: usize,
141    /// Maximum error rate (0.0 to 1.0)
142    pub max_error_rate: f64,
143    /// Maximum memory usage in MB
144    pub max_memory_usage_mb: f64,
145    /// Minimum throughput (updates per second)
146    pub min_throughput_ups: f64,
147}
148
149impl Default for AlertThresholds {
150    fn default() -> Self {
151        Self {
152            max_processing_latency_ms: 1000,
153            max_queue_depth: 10000,
154            max_error_rate: 0.05,
155            max_memory_usage_mb: 1024.0,
156            min_throughput_ups: 100.0,
157        }
158    }
159}
160
161/// Version control configuration
162#[derive(Debug, Clone, Serialize, Deserialize)]
163pub struct VersionControlConfig {
164    /// Enable versioning
165    pub enable_versioning: bool,
166    /// Maximum versions to keep
167    pub max_versions: usize,
168    /// Version compression threshold
169    pub compression_threshold: usize,
170    /// Enable incremental versioning
171    pub enable_incremental: bool,
172}
173
174impl Default for VersionControlConfig {
175    fn default() -> Self {
176        Self {
177            enable_versioning: true,
178            max_versions: 10,
179            compression_threshold: 1000,
180            enable_incremental: true,
181        }
182    }
183}
184
185/// Quality assurance configuration
186#[derive(Debug, Clone, Serialize, Deserialize)]
187pub struct QualityAssuranceConfig {
188    /// Enable quality checking
189    pub enable_quality_checks: bool,
190    /// Quality threshold (0.0 to 1.0)
191    pub quality_threshold: f64,
192    /// Enable anomaly detection
193    pub enable_anomaly_detection: bool,
194    /// Anomaly sensitivity (0.0 to 1.0)
195    pub anomaly_sensitivity: f64,
196}
197
198impl Default for QualityAssuranceConfig {
199    fn default() -> Self {
200        Self {
201            enable_quality_checks: true,
202            quality_threshold: 0.8,
203            enable_anomaly_detection: true,
204            anomaly_sensitivity: 0.7,
205        }
206    }
207}
208
209/// Auto-scaling configuration
210#[derive(Debug, Clone, Serialize, Deserialize)]
211pub struct AutoScalingConfig {
212    /// Enable auto-scaling
213    pub enable_autoscaling: bool,
214    /// Minimum number of workers
215    pub min_workers: usize,
216    /// Maximum number of workers
217    pub max_workers: usize,
218    /// CPU threshold for scaling up
219    pub scale_up_cpu_threshold: f64,
220    /// CPU threshold for scaling down
221    pub scale_down_cpu_threshold: f64,
222    /// Memory threshold for scaling up
223    pub scale_up_memory_threshold: f64,
224    /// Queue depth threshold for scaling up
225    pub scale_up_queue_threshold: usize,
226    /// Cooldown period between scaling operations (seconds)
227    pub scaling_cooldown_seconds: u64,
228}
229
230impl Default for AutoScalingConfig {
231    fn default() -> Self {
232        Self {
233            enable_autoscaling: true,
234            min_workers: 2,
235            max_workers: 20,
236            scale_up_cpu_threshold: 0.8,
237            scale_down_cpu_threshold: 0.3,
238            scale_up_memory_threshold: 0.8,
239            scale_up_queue_threshold: 1000,
240            scaling_cooldown_seconds: 300,
241        }
242    }
243}
244
245/// Compression configuration for vector storage
246#[derive(Debug, Clone, Serialize, Deserialize)]
247pub struct CompressionConfig {
248    /// Enable vector compression
249    pub enable_compression: bool,
250    /// Compression method
251    pub compression_method: CompressionMethod,
252    /// Compression level (0-9)
253    pub compression_level: u8,
254    /// Compression threshold (minimum vector size to compress)
255    pub compression_threshold: usize,
256}
257
258impl Default for CompressionConfig {
259    fn default() -> Self {
260        Self {
261            enable_compression: true,
262            compression_method: CompressionMethod::Quantization,
263            compression_level: 6,
264            compression_threshold: 100,
265        }
266    }
267}
268
269/// Compression methods for vectors
270#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
271pub enum CompressionMethod {
272    /// Product quantization
273    ProductQuantization,
274    /// Scalar quantization
275    ScalarQuantization,
276    /// General quantization
277    Quantization,
278    /// PCA compression
279    PCA,
280    /// Dictionary compression
281    Dictionary,
282}
oxirs_vec/real_time_embedding_pipeline/config.rs

oxirs_vec/real_time_embedding_pipeline/
config.rs
