optirs_core/streaming/adaptive_streaming/
config.rs

1// Configuration management for adaptive streaming optimization
2//
3// This module provides comprehensive configuration types and settings for
4// streaming optimization scenarios, including drift detection parameters,
5// performance tracking settings, resource allocation strategies, and
6// adaptive buffer management configurations.
7
8use serde::{Deserialize, Serialize};
9use std::time::Duration;
10
11/// Main configuration for adaptive streaming optimization
12#[derive(Debug, Clone, Serialize, Deserialize, Default)]
13pub struct StreamingConfig {
14    /// Buffer management configuration
15    pub buffer_config: BufferConfig,
16    /// Drift detection configuration
17    pub drift_config: DriftConfig,
18    /// Performance tracking configuration
19    pub performance_config: PerformanceConfig,
20    /// Resource management configuration
21    pub resource_config: ResourceConfig,
22    /// Meta-learning configuration
23    pub meta_learning_config: MetaLearningConfig,
24    /// Anomaly detection configuration
25    pub anomaly_config: AnomalyConfig,
26    /// Learning rate adaptation configuration
27    pub learning_rate_config: LearningRateConfig,
28}
29
30/// Buffer management configuration
31#[derive(Debug, Clone, Serialize, Deserialize)]
32pub struct BufferConfig {
33    /// Initial buffer size
34    pub initial_size: usize,
35    /// Maximum buffer size
36    pub max_size: usize,
37    /// Minimum buffer size
38    pub min_size: usize,
39    /// Buffer size adaptation strategy
40    pub size_strategy: BufferSizeStrategy,
41    /// Quality threshold for buffer processing
42    pub quality_threshold: f64,
43    /// Enable adaptive buffer sizing
44    pub enable_adaptive_sizing: bool,
45    /// Buffer processing timeout
46    pub processing_timeout: Duration,
47    /// Memory limit for buffer in MB
48    pub memory_limit_mb: usize,
49}
50
51impl Default for BufferConfig {
52    fn default() -> Self {
53        Self {
54            initial_size: 1000,
55            max_size: 10000,
56            min_size: 100,
57            size_strategy: BufferSizeStrategy::Adaptive,
58            quality_threshold: 0.8,
59            enable_adaptive_sizing: true,
60            processing_timeout: Duration::from_secs(30),
61            memory_limit_mb: 512,
62        }
63    }
64}
65
66/// Buffer sizing strategies
67#[derive(Debug, Clone, Serialize, Deserialize)]
68pub enum BufferSizeStrategy {
69    /// Fixed buffer size
70    Fixed,
71    /// Linear growth/shrinkage
72    Linear { growth_rate: f64 },
73    /// Exponential growth/shrinkage
74    Exponential { base: f64 },
75    /// Adaptive based on performance
76    Adaptive,
77    /// Based on resource availability
78    ResourceBased,
79}
80
81/// Drift detection configuration
82#[derive(Debug, Clone, Serialize, Deserialize)]
83pub struct DriftConfig {
84    /// Enable drift detection
85    pub enable_detection: bool,
86    /// Drift detection method
87    pub detection_method: DriftDetectionMethod,
88    /// Sensitivity for drift detection
89    pub sensitivity: f64,
90    /// Minimum samples before drift detection
91    pub min_samples: usize,
92    /// Warning threshold for gradual drift
93    pub warning_threshold: f64,
94    /// Drift threshold for concept drift
95    pub drift_threshold: f64,
96    /// Window size for statistical tests
97    pub window_size: usize,
98    /// Enable false positive tracking
99    pub enable_false_positive_tracking: bool,
100    /// Statistical significance level
101    pub significance_level: f64,
102    /// Adaptation speed after drift detection
103    pub adaptation_speed: f64,
104}
105
106impl Default for DriftConfig {
107    fn default() -> Self {
108        Self {
109            enable_detection: true,
110            detection_method: DriftDetectionMethod::Statistical(StatisticalMethod::ADWIN),
111            sensitivity: 0.05,
112            min_samples: 30,
113            warning_threshold: 0.8,
114            drift_threshold: 1.2,
115            window_size: 1000,
116            enable_false_positive_tracking: true,
117            significance_level: 0.05,
118            adaptation_speed: 0.1,
119        }
120    }
121}
122
123/// Drift detection methods
124#[derive(Debug, Clone, Serialize, Deserialize)]
125pub enum DriftDetectionMethod {
126    /// Statistical methods for drift detection
127    Statistical(StatisticalMethod),
128    /// Distribution-based methods
129    Distribution(DistributionMethod),
130    /// Model-based drift detection
131    ModelBased(ModelType),
132    /// Ensemble of multiple methods
133    Ensemble {
134        methods: Vec<DriftDetectionMethod>,
135        voting_strategy: VotingStrategy,
136    },
137}
138
139/// Statistical methods for drift detection
140#[derive(Debug, Clone, Serialize, Deserialize, Hash, Eq, PartialEq)]
141pub enum StatisticalMethod {
142    /// Adaptive Windowing (ADWIN)
143    ADWIN,
144    /// Drift Detection Method (DDM)
145    DDM,
146    /// Early Drift Detection Method (EDDM)
147    EDDM,
148    /// Page Hinkley test
149    PageHinkley,
150    /// CUSUM test
151    CUSUM,
152    /// Kolmogorov-Smirnov test
153    KolmogorovSmirnov,
154    /// Mann-Whitney U test
155    MannWhitneyU,
156}
157
158/// Distribution-based drift detection methods
159#[derive(Debug, Clone, Serialize, Deserialize, Hash, Eq, PartialEq)]
160pub enum DistributionMethod {
161    /// Kullback-Leibler divergence
162    KLDivergence,
163    /// Jensen-Shannon divergence
164    JSDivergence,
165    /// Hellinger distance
166    HellingerDistance,
167    /// Wasserstein distance
168    WassersteinDistance,
169    /// Earth Mover's Distance
170    EarthMoverDistance,
171}
172
173/// Model types for drift detection
174#[derive(Debug, Clone, Serialize, Deserialize, Hash, Eq, PartialEq)]
175pub enum ModelType {
176    /// Linear model comparison
177    Linear,
178    /// Neural network based
179    NeuralNetwork,
180    /// Decision tree based
181    DecisionTree,
182    /// Ensemble methods
183    Ensemble,
184}
185
186/// Voting strategies for ensemble drift detection
187#[derive(Debug, Clone, Serialize, Deserialize)]
188pub enum VotingStrategy {
189    /// Majority voting
190    Majority,
191    /// Weighted voting
192    Weighted { weights: Vec<f64> },
193    /// Unanimous decision
194    Unanimous,
195    /// Threshold-based (minimum number of votes)
196    Threshold { min_votes: usize },
197}
198
199/// Performance tracking configuration
200#[derive(Debug, Clone, Serialize, Deserialize)]
201pub struct PerformanceConfig {
202    /// Enable performance tracking
203    pub enable_tracking: bool,
204    /// Performance metrics to track
205    pub metrics: Vec<PerformanceMetricType>,
206    /// Performance history size
207    pub history_size: usize,
208    /// Performance evaluation frequency
209    pub evaluation_frequency: usize,
210    /// Enable trend analysis
211    pub enable_trend_analysis: bool,
212    /// Trend window size
213    pub trend_window_size: usize,
214    /// Enable performance prediction
215    pub enable_prediction: bool,
216    /// Prediction horizon (steps ahead)
217    pub prediction_horizon: usize,
218    /// Performance baseline update frequency
219    pub baseline_update_frequency: usize,
220}
221
222impl Default for PerformanceConfig {
223    fn default() -> Self {
224        Self {
225            enable_tracking: true,
226            metrics: vec![
227                PerformanceMetricType::Loss,
228                PerformanceMetricType::Accuracy,
229                PerformanceMetricType::Convergence,
230                PerformanceMetricType::ResourceUsage,
231            ],
232            history_size: 1000,
233            evaluation_frequency: 10,
234            enable_trend_analysis: true,
235            trend_window_size: 100,
236            enable_prediction: true,
237            prediction_horizon: 10,
238            baseline_update_frequency: 100,
239        }
240    }
241}
242
243/// Types of performance metrics to track
244#[derive(Debug, Clone, Serialize, Deserialize)]
245pub enum PerformanceMetricType {
246    /// Loss function value
247    Loss,
248    /// Accuracy metric
249    Accuracy,
250    /// Convergence rate
251    Convergence,
252    /// Resource usage efficiency
253    ResourceUsage,
254    /// Gradient norm
255    GradientNorm,
256    /// Parameter updates magnitude
257    ParameterUpdates,
258    /// Learning rate effectiveness
259    LearningRateEffectiveness,
260    /// Custom metric
261    Custom(String),
262}
263
264/// Resource management configuration
265#[derive(Debug, Clone, Serialize, Deserialize)]
266pub struct ResourceConfig {
267    /// Maximum memory usage in MB
268    pub max_memory_mb: usize,
269    /// Maximum CPU usage percentage
270    pub max_cpu_percent: f64,
271    /// Resource allocation strategy
272    pub allocation_strategy: ResourceAllocationStrategy,
273    /// Enable dynamic resource allocation
274    pub enable_dynamic_allocation: bool,
275    /// Resource monitoring frequency
276    pub monitoring_frequency: Duration,
277    /// Resource budget constraints
278    pub budget_constraints: ResourceBudgetConstraints,
279    /// Enable resource prediction
280    pub enable_resource_prediction: bool,
281    /// Resource cleanup threshold
282    pub cleanup_threshold: f64,
283}
284
285impl Default for ResourceConfig {
286    fn default() -> Self {
287        Self {
288            max_memory_mb: 2048,
289            max_cpu_percent: 80.0,
290            allocation_strategy: ResourceAllocationStrategy::Adaptive,
291            enable_dynamic_allocation: true,
292            monitoring_frequency: Duration::from_secs(10),
293            budget_constraints: ResourceBudgetConstraints::default(),
294            enable_resource_prediction: true,
295            cleanup_threshold: 0.9,
296        }
297    }
298}
299
300/// Resource allocation strategies
301#[derive(Debug, Clone, Serialize, Deserialize)]
302pub enum ResourceAllocationStrategy {
303    /// Static allocation
304    Static,
305    /// Dynamic allocation based on demand
306    Dynamic,
307    /// Adaptive allocation based on performance
308    Adaptive,
309    /// Proportional allocation
310    Proportional,
311    /// Priority-based allocation
312    PriorityBased,
313}
314
315/// Resource budget constraints
316#[derive(Debug, Clone, Serialize, Deserialize)]
317pub struct ResourceBudgetConstraints {
318    /// Memory budget in MB
319    pub memory_budget_mb: usize,
320    /// CPU budget as percentage
321    pub cpu_budget_percent: f64,
322    /// Time budget per operation
323    pub time_budget: Duration,
324    /// Enable strict budget enforcement
325    pub strict_enforcement: bool,
326    /// Budget violation penalty
327    pub violation_penalty: f64,
328}
329
330impl Default for ResourceBudgetConstraints {
331    fn default() -> Self {
332        Self {
333            memory_budget_mb: 1024,
334            cpu_budget_percent: 60.0,
335            time_budget: Duration::from_secs(60),
336            strict_enforcement: false,
337            violation_penalty: 0.1,
338        }
339    }
340}
341
342/// Meta-learning configuration
343#[derive(Debug, Clone, Serialize, Deserialize)]
344pub struct MetaLearningConfig {
345    /// Enable meta-learning
346    pub enable_meta_learning: bool,
347    /// Meta-learning algorithm
348    pub algorithm: MetaAlgorithm,
349    /// Experience buffer size
350    pub experience_buffer_size: usize,
351    /// Meta-learning update frequency
352    pub update_frequency: usize,
353    /// Learning rate for meta-learning
354    pub meta_learning_rate: f64,
355    /// Exploration rate for meta-learning
356    pub exploration_rate: f64,
357    /// Meta-model complexity
358    pub model_complexity: MetaModelComplexity,
359    /// Enable transfer learning
360    pub enable_transfer_learning: bool,
361    /// Experience replay configuration
362    pub replay_config: ExperienceReplayConfig,
363}
364
365impl Default for MetaLearningConfig {
366    fn default() -> Self {
367        Self {
368            enable_meta_learning: true,
369            algorithm: MetaAlgorithm::ModelAgnosticMetaLearning,
370            experience_buffer_size: 10000,
371            update_frequency: 100,
372            meta_learning_rate: 0.001,
373            exploration_rate: 0.1,
374            model_complexity: MetaModelComplexity::Medium,
375            enable_transfer_learning: true,
376            replay_config: ExperienceReplayConfig::default(),
377        }
378    }
379}
380
381/// Meta-learning algorithms
382#[derive(Debug, Clone, Serialize, Deserialize)]
383pub enum MetaAlgorithm {
384    /// Model-Agnostic Meta-Learning (MAML)
385    ModelAgnosticMetaLearning,
386    /// Learning to Learn by Gradient Descent by Gradient Descent
387    LearningToLearn,
388    /// Meta-SGD
389    MetaSGD,
390    /// Reptile
391    Reptile,
392    /// Online Meta-Learning
393    OnlineMetaLearning,
394    /// Continual Meta-Learning
395    ContinualMetaLearning,
396}
397
398/// Meta-model complexity levels
399#[derive(Debug, Clone, Serialize, Deserialize)]
400pub enum MetaModelComplexity {
401    /// Low complexity (simple linear models)
402    Low,
403    /// Medium complexity (small neural networks)
404    Medium,
405    /// High complexity (large neural networks)
406    High,
407    /// Adaptive complexity based on performance
408    Adaptive,
409}
410
411/// Experience replay configuration
412#[derive(Debug, Clone, Serialize, Deserialize)]
413pub struct ExperienceReplayConfig {
414    /// Enable prioritized experience replay
415    pub enable_prioritized_replay: bool,
416    /// Priority calculation method
417    pub priority_method: PriorityMethod,
418    /// Replay batch size
419    pub batch_size: usize,
420    /// Replay frequency
421    pub replay_frequency: usize,
422    /// Experience importance sampling
423    pub importance_sampling: bool,
424}
425
426impl Default for ExperienceReplayConfig {
427    fn default() -> Self {
428        Self {
429            enable_prioritized_replay: true,
430            priority_method: PriorityMethod::TDError,
431            batch_size: 32,
432            replay_frequency: 10,
433            importance_sampling: true,
434        }
435    }
436}
437
438/// Priority calculation methods for experience replay
439#[derive(Debug, Clone, Serialize, Deserialize)]
440pub enum PriorityMethod {
441    /// Temporal Difference error
442    TDError,
443    /// Surprise-based priority
444    Surprise,
445    /// Gradient magnitude
446    GradientMagnitude,
447    /// Loss improvement
448    LossImprovement,
449    /// Random priority
450    Random,
451}
452
453/// Anomaly detection configuration
454#[derive(Debug, Clone, Serialize, Deserialize)]
455pub struct AnomalyConfig {
456    /// Enable anomaly detection
457    pub enable_detection: bool,
458    /// Anomaly detection method
459    pub detection_method: AnomalyDetectionMethod,
460    /// Anomaly threshold
461    pub threshold: f64,
462    /// Window size for anomaly detection
463    pub window_size: usize,
464    /// Enable adaptive thresholding
465    pub enable_adaptive_threshold: bool,
466    /// False positive rate tolerance
467    pub false_positive_rate: f64,
468    /// Contamination rate assumption
469    pub contamination_rate: f64,
470    /// Response strategy for detected anomalies
471    pub response_strategy: AnomalyResponseStrategy,
472}
473
474impl Default for AnomalyConfig {
475    fn default() -> Self {
476        Self {
477            enable_detection: true,
478            detection_method: AnomalyDetectionMethod::StatisticalOutlier,
479            threshold: 2.0,
480            window_size: 100,
481            enable_adaptive_threshold: true,
482            false_positive_rate: 0.05,
483            contamination_rate: 0.1,
484            response_strategy: AnomalyResponseStrategy::Adaptive,
485        }
486    }
487}
488
489/// Anomaly detection methods
490#[derive(Debug, Clone, Serialize, Deserialize)]
491pub enum AnomalyDetectionMethod {
492    /// Statistical outlier detection
493    StatisticalOutlier,
494    /// Isolation Forest
495    IsolationForest,
496    /// One-Class SVM
497    OneClassSVM,
498    /// Local Outlier Factor
499    LocalOutlierFactor,
500    /// Autoencoders
501    Autoencoder,
502    /// Ensemble methods
503    Ensemble,
504}
505
506/// Response strategies for detected anomalies
507#[derive(Debug, Clone, Serialize, Deserialize)]
508pub enum AnomalyResponseStrategy {
509    /// Ignore anomalies
510    Ignore,
511    /// Filter out anomalous data
512    Filter,
513    /// Adapt model to handle anomalies
514    Adaptive,
515    /// Reset to safe state
516    Reset,
517    /// Custom response strategy
518    Custom(String),
519}
520
521/// Learning rate adaptation configuration
522#[derive(Debug, Clone, Serialize, Deserialize)]
523pub struct LearningRateConfig {
524    /// Initial learning rate
525    pub initial_rate: f64,
526    /// Minimum learning rate
527    pub min_rate: f64,
528    /// Maximum learning rate
529    pub max_rate: f64,
530    /// Learning rate adaptation strategy
531    pub adaptation_strategy: LearningRateAdaptationStrategy,
532    /// Adaptation frequency
533    pub adaptation_frequency: usize,
534    /// Performance sensitivity for adaptation
535    pub performance_sensitivity: f64,
536    /// Enable cyclical learning rates
537    pub enable_cyclical_rates: bool,
538    /// Cycle configuration
539    pub cycle_config: CyclicalRateConfig,
540}
541
542impl Default for LearningRateConfig {
543    fn default() -> Self {
544        Self {
545            initial_rate: 0.001,
546            min_rate: 1e-6,
547            max_rate: 0.1,
548            adaptation_strategy: LearningRateAdaptationStrategy::PerformanceBased,
549            adaptation_frequency: 10,
550            performance_sensitivity: 0.1,
551            enable_cyclical_rates: false,
552            cycle_config: CyclicalRateConfig::default(),
553        }
554    }
555}
556
557/// Learning rate adaptation strategies
558#[derive(Debug, Clone, Serialize, Deserialize)]
559pub enum LearningRateAdaptationStrategy {
560    /// Fixed learning rate
561    Fixed,
562    /// Step decay
563    StepDecay { decay_rate: f64, decay_steps: usize },
564    /// Exponential decay
565    ExponentialDecay { decay_rate: f64 },
566    /// Performance-based adaptation
567    PerformanceBased,
568    /// Gradient-based adaptation
569    GradientBased,
570    /// Adaptive learning rate (AdaGrad-style)
571    Adaptive,
572    /// Cosine annealing
573    CosineAnnealing { t_max: usize },
574}
575
576/// Cyclical learning rate configuration
577#[derive(Debug, Clone, Serialize, Deserialize)]
578pub struct CyclicalRateConfig {
579    /// Base learning rate
580    pub base_rate: f64,
581    /// Maximum learning rate in cycle
582    pub max_rate: f64,
583    /// Cycle length (number of steps)
584    pub cycle_length: usize,
585    /// Cycle mode
586    pub cycle_mode: CycleMode,
587    /// Scale function for cycle
588    pub scale_function: ScaleFunction,
589}
590
591impl Default for CyclicalRateConfig {
592    fn default() -> Self {
593        Self {
594            base_rate: 0.0001,
595            max_rate: 0.001,
596            cycle_length: 1000,
597            cycle_mode: CycleMode::Triangular,
598            scale_function: ScaleFunction::Linear,
599        }
600    }
601}
602
603/// Cyclical learning rate cycle modes
604#[derive(Debug, Clone, Serialize, Deserialize)]
605pub enum CycleMode {
606    /// Triangular cycle
607    Triangular,
608    /// Triangular2 (amplitude scales)
609    Triangular2,
610    /// Exponential range
611    ExponentialRange,
612    /// Custom cycle pattern
613    Custom(String),
614}
615
616/// Scale functions for cyclical learning rates
617#[derive(Debug, Clone, Serialize, Deserialize)]
618pub enum ScaleFunction {
619    /// Linear scaling
620    Linear,
621    /// Exponential scaling
622    Exponential { factor: f64 },
623    /// Polynomial scaling
624    Polynomial { power: f64 },
625    /// Custom scaling function
626    Custom(String),
627}
628
629/// Validation methods for configuration
630impl StreamingConfig {
631    /// Validates the configuration for consistency and feasibility
632    pub fn validate(&self) -> Result<(), String> {
633        // Validate buffer configuration
634        if self.buffer_config.max_size < self.buffer_config.min_size {
635            return Err("Buffer max_size must be >= min_size".to_string());
636        }
637
638        if self.buffer_config.initial_size < self.buffer_config.min_size
639            || self.buffer_config.initial_size > self.buffer_config.max_size
640        {
641            return Err("Buffer initial_size must be between min_size and max_size".to_string());
642        }
643
644        // Validate drift configuration
645        if self.drift_config.sensitivity <= 0.0 || self.drift_config.sensitivity > 1.0 {
646            return Err("Drift sensitivity must be in (0, 1]".to_string());
647        }
648
649        if self.drift_config.warning_threshold >= self.drift_config.drift_threshold {
650            return Err("Drift warning_threshold must be < drift_threshold".to_string());
651        }
652
653        // Validate learning rate configuration
654        if self.learning_rate_config.min_rate >= self.learning_rate_config.max_rate {
655            return Err("Learning rate min_rate must be < max_rate".to_string());
656        }
657
658        if self.learning_rate_config.initial_rate < self.learning_rate_config.min_rate
659            || self.learning_rate_config.initial_rate > self.learning_rate_config.max_rate
660        {
661            return Err(
662                "Learning rate initial_rate must be between min_rate and max_rate".to_string(),
663            );
664        }
665
666        // Validate resource configuration
667        if self.resource_config.max_cpu_percent <= 0.0
668            || self.resource_config.max_cpu_percent > 100.0
669        {
670            return Err("Resource max_cpu_percent must be in (0, 100]".to_string());
671        }
672
673        // Validate meta-learning configuration
674        if self.meta_learning_config.meta_learning_rate <= 0.0 {
675            return Err("Meta-learning rate must be > 0".to_string());
676        }
677
678        if self.meta_learning_config.exploration_rate < 0.0
679            || self.meta_learning_config.exploration_rate > 1.0
680        {
681            return Err("Meta-learning exploration_rate must be in [0, 1]".to_string());
682        }
683
684        Ok(())
685    }
686
687    /// Creates a configuration optimized for low-latency streaming
688    pub fn low_latency() -> Self {
689        let mut config = Self::default();
690        config.buffer_config.initial_size = 100;
691        config.buffer_config.max_size = 1000;
692        config.buffer_config.processing_timeout = Duration::from_millis(100);
693        config.performance_config.evaluation_frequency = 5;
694        config.drift_config.min_samples = 10;
695        config.resource_config.monitoring_frequency = Duration::from_secs(1);
696        config
697    }
698
699    /// Creates a configuration optimized for high-throughput streaming
700    pub fn high_throughput() -> Self {
701        let mut config = Self::default();
702        config.buffer_config.initial_size = 5000;
703        config.buffer_config.max_size = 50000;
704        config.buffer_config.memory_limit_mb = 2048;
705        config.performance_config.evaluation_frequency = 100;
706        config.drift_config.window_size = 5000;
707        config.resource_config.max_memory_mb = 4096;
708        config
709    }
710
711    /// Creates a configuration optimized for memory-constrained environments
712    pub fn memory_efficient() -> Self {
713        let mut config = Self::default();
714        config.buffer_config.initial_size = 200;
715        config.buffer_config.max_size = 2000;
716        config.buffer_config.memory_limit_mb = 128;
717        config.performance_config.history_size = 100;
718        config.meta_learning_config.experience_buffer_size = 1000;
719        config.resource_config.max_memory_mb = 256;
720        config.drift_config.window_size = 500;
721        config
722    }
723}
optirs_core/streaming/adaptive_streaming/config.rs

optirs_core/streaming/adaptive_streaming/
config.rs
