optirs_core/streaming/

enhanced_adaptive_lr.rs

// Enhanced adaptive learning rate mechanisms for streaming optimization
//
// This module provides advanced adaptive learning rate controllers that can
// dynamically adjust learning rates based on multiple signals including
// gradient statistics, performance metrics, concept drift, and resource constraints.

use scirs2_core::ndarray::{Array1, Array2};
use scirs2_core::numeric::Float;
use std::collections::{HashMap, VecDeque};
use std::time::{Duration, Instant};

#[allow(unused_imports)]
use crate::error::Result;

/// Performance metric types for adaptation
#[derive(Debug, Clone)]
pub enum PerformanceMetric<A: Float + Send + Sync> {
    Loss(A),
    Accuracy(A),
    F1Score(A),
    AUC(A),
    Custom { name: String, value: A },
}

/// Enhanced adaptive learning rate controller with multiple adaptation mechanisms
#[derive(Debug, Clone)]
pub struct EnhancedAdaptiveLRController<A: Float + Send + Sync> {
    /// Current learning rate
    current_lr: A,

    /// Base learning rate
    base_lr: A,

    /// Learning rate bounds
    min_lr: A,
    max_lr: A,

    /// Multi-signal adaptation strategy
    adaptation_strategy: MultiSignalAdaptationStrategy<A>,

    /// Gradient-based adaptation state
    gradient_adapter: GradientBasedAdapter<A>,

    /// Performance-based adaptation state
    performance_adapter: PerformanceBasedAdapter<A>,

    /// Drift-aware adaptation
    drift_adapter: DriftAwareAdapter<A>,

    /// Resource-aware adaptation
    resource_adapter: ResourceAwareAdapter<A>,

    /// Meta-learning for hyperparameter optimization
    meta_optimizer: MetaOptimizer<A>,

    /// Adaptation history for analysis
    adaptation_history: VecDeque<AdaptationEvent<A>>,

    /// Configuration
    config: AdaptiveLRConfig<A>,
}

/// Configuration for adaptive learning rate controller
#[derive(Debug, Clone)]
pub struct AdaptiveLRConfig<A: Float + Send + Sync> {
    /// Base learning rate
    pub base_lr: A,

    /// Minimum allowed learning rate
    pub min_lr: A,

    /// Maximum allowed learning rate  
    pub max_lr: A,

    /// Enable gradient-based adaptation
    pub enable_gradient_adaptation: bool,

    /// Enable performance-based adaptation
    pub enable_performance_adaptation: bool,

    /// Enable drift-aware adaptation
    pub enable_drift_adaptation: bool,

    /// Enable resource-aware adaptation
    pub enable_resource_adaptation: bool,

    /// Enable meta-learning optimization
    pub enable_meta_learning: bool,

    /// History window size
    pub history_window_size: usize,

    /// Adaptation frequency (steps)
    pub adaptation_frequency: usize,

    /// Sensitivity to changes
    pub adaptation_sensitivity: A,

    /// Use ensemble voting for conflicting signals
    pub use_ensemble_voting: bool,
}

/// Multi-signal adaptation strategy
#[derive(Debug, Clone)]
pub struct MultiSignalAdaptationStrategy<A: Float + Send + Sync> {
    /// Weighted voting system for adaptation signals
    signal_weights: HashMap<AdaptationSignalType, A>,

    /// Signal voting history
    voting_history: VecDeque<SignalVote<A>>,

    /// Conflict resolution method
    conflict_resolution: ConflictResolution,

    /// Signal reliability scores
    signal_reliability: HashMap<AdaptationSignalType, A>,

    /// Last adaptation decision
    last_decision: Option<AdaptationDecision<A>>,
}

/// Types of adaptation signals
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum AdaptationSignalType {
    GradientMagnitude,
    GradientVariance,
    LossProgression,
    AccuracyTrend,
    ConceptDrift,
    ResourceUtilization,
    ModelComplexity,
    DataQuality,
}

/// Signal vote for learning rate adaptation
#[derive(Debug, Clone)]
pub struct SignalVote<A: Float + Send + Sync> {
    signal_type: AdaptationSignalType,
    recommended_lr_change: A, // Multiplier (1.0 = no change)
    confidence: A,
    reasoning: String,
    timestamp: Instant,
}

/// Conflict resolution methods for contradictory signals
#[derive(Debug, Clone, Copy)]
pub enum ConflictResolution {
    /// Use weighted average of all signals
    WeightedAverage,
    /// Use signal with highest confidence
    HighestConfidence,
    /// Use majority vote (requires threshold)
    MajorityVote { threshold: f64 },
    /// Use conservative approach (smallest change)
    Conservative,
    /// Use meta-learning to resolve conflicts
    MetaLearned,
}

/// Adaptation decision with rationale
#[derive(Debug, Clone)]
pub struct AdaptationDecision<A: Float + Send + Sync> {
    new_lr: A,
    lr_multiplier: A,
    contributing_signals: Vec<AdaptationSignalType>,
    confidence: A,
    rationale: String,
    timestamp: Instant,
}

/// Gradient-based adaptation using statistical analysis
#[derive(Debug, Clone)]
pub struct GradientBasedAdapter<A: Float + Send + Sync> {
    /// Gradient magnitude history
    magnitude_history: VecDeque<A>,

    /// Gradient direction variance
    direction_variance_history: VecDeque<A>,

    /// Gradient norm statistics
    norm_statistics: GradientNormStatistics<A>,

    /// Signal-to-noise ratio estimation
    snr_estimator: SignalToNoiseEstimator<A>,

    /// Gradient staleness detection
    staleness_detector: GradientStalenessDetector<A>,
}

/// Performance-based adaptation using multiple metrics
#[derive(Debug, Clone)]
pub struct PerformanceBasedAdapter<A: Float + Send + Sync> {
    /// Performance metric history
    metric_history: HashMap<String, VecDeque<A>>,

    /// Performance trend analysis
    trend_analyzer: PerformanceTrendAnalyzer<A>,

    /// Plateau detection
    plateau_detector: PlateauDetector<A>,

    /// Overfitting detection
    overfitting_detector: OverfittingDetector<A>,

    /// Learning efficiency tracker
    efficiency_tracker: LearningEfficiencyTracker<A>,
}

/// Drift-aware adaptation for non-stationary data
#[derive(Debug, Clone)]
pub struct DriftAwareAdapter<A: Float + Send + Sync> {
    /// Concept drift detection methods
    drift_detectors: Vec<ConceptDriftDetector<A>>,

    /// Data distribution shift detection
    distribution_tracker: DistributionTracker<A>,

    /// Adaptation speed controller
    adaptation_speed: AdaptationSpeedController<A>,

    /// Drift severity assessment
    drift_severity: DriftSeverityAssessor<A>,
}

/// Resource-aware adaptation based on computational constraints
#[derive(Debug, Clone)]
pub struct ResourceAwareAdapter<A: Float + Send + Sync> {
    /// Memory usage tracker
    memory_tracker: MemoryUsageTracker,

    /// Computation time tracker
    compute_tracker: ComputationTimeTracker,

    /// Energy consumption tracker
    energy_tracker: EnergyConsumptionTracker,

    /// Throughput requirements
    throughput_requirements: ThroughputRequirements<A>,

    /// Resource budget manager
    budget_manager: ResourceBudgetManager<A>,
}

/// Meta-learning optimizer for hyperparameter adaptation
#[derive(Debug, Clone)]
pub struct MetaOptimizer<A: Float + Send + Sync> {
    /// Neural network for learning rate prediction
    lr_predictor: LearningRatePredictorNetwork<A>,

    /// Hyperparameter optimization history
    optimization_history: VecDeque<HyperparameterUpdate<A>>,

    /// Multi-armed bandit for exploration
    exploration_strategy: ExplorationStrategy<A>,

    /// Transfer learning from similar tasks
    transfer_learner: TransferLearner<A>,
}

/// Adaptation event for tracking and analysis
#[derive(Debug, Clone)]
pub struct AdaptationEvent<A: Float + Send + Sync> {
    timestamp: Instant,
    old_lr: A,
    new_lr: A,
    trigger_signals: Vec<AdaptationSignalType>,
    adaptation_reason: String,
    confidence: A,
    effectiveness_score: Option<A>, // Measured retrospectively
}

/// Gradient norm statistics for adaptation
#[derive(Debug, Clone)]
pub struct GradientNormStatistics<A: Float + Send + Sync> {
    mean: A,
    variance: A,
    skewness: A,
    kurtosis: A,
    percentiles: Vec<A>, // 5th, 25th, 50th, 75th, 95th
    autocorrelation: A,
}

/// Signal-to-noise ratio estimation for gradients
#[derive(Debug, Clone)]
pub struct SignalToNoiseEstimator<A: Float + Send + Sync> {
    signal_estimate: A,
    noise_estimate: A,
    snr_history: VecDeque<A>,
    estimation_method: SNREstimationMethod,
}

#[derive(Debug, Clone, Copy)]
pub enum SNREstimationMethod {
    MovingAverage,
    ExponentialSmoothing,
    RobustEstimation,
    WaveletDenoising,
}

/// Gradient staleness detection for distributed settings
#[derive(Debug, Clone)]
pub struct GradientStalenessDetector<A: Float + Send + Sync> {
    staleness_threshold: Duration,
    gradient_timestamps: VecDeque<Instant>,
    staleness_impact_model: StalenessImpactModel<A>,
}

#[derive(Debug, Clone)]
pub struct StalenessImpactModel<A: Float + Send + Sync> {
    staleness_penalty: A,
    compensation_factor: A,
    impact_history: VecDeque<A>,
}

/// Performance trend analysis for learning rate adaptation
#[derive(Debug, Clone)]
pub struct PerformanceTrendAnalyzer<A: Float + Send + Sync> {
    trend_detection_window: usize,
    trend_types: Vec<TrendType>,
    trend_strength: A,
    trend_duration: Duration,
}

#[derive(Debug, Clone, Copy)]
pub enum TrendType {
    Improving,
    Degrading,
    Oscillating,
    Plateau,
    Volatile,
}

/// Plateau detection in learning curves
#[derive(Debug, Clone)]
pub struct PlateauDetector<A: Float + Send + Sync> {
    plateau_threshold: A,
    min_plateau_duration: usize,
    current_plateau_length: usize,
    plateau_confidence: A,
}

/// Overfitting detection mechanism
#[derive(Debug, Clone)]
pub struct OverfittingDetector<A: Float + Send + Sync> {
    train_loss_history: VecDeque<A>,
    val_loss_history: VecDeque<A>,
    overfitting_threshold: A,
    early_stopping_patience: usize,
}

/// Learning efficiency tracking
#[derive(Debug, Clone)]
pub struct LearningEfficiencyTracker<A: Float + Send + Sync> {
    loss_reduction_per_step: VecDeque<A>,
    parameter_change_magnitude: VecDeque<A>,
    efficiency_score: A,
    efficiency_trend: TrendType,
}

/// Concept drift detection methods
#[derive(Debug, Clone)]
pub struct ConceptDriftDetector<A: Float + Send + Sync> {
    detection_method: DriftDetectionMethod,
    drift_threshold: A,
    window_size: usize,
    drift_confidence: A,
    last_drift_time: Option<Instant>,
}

#[derive(Debug, Clone, Copy)]
pub enum DriftDetectionMethod {
    ADWIN,
    DDM,
    EDDM,
    PageHinkley,
    KSWIN,
    Statistical,
}

/// Data distribution tracking
#[derive(Debug, Clone)]
pub struct DistributionTracker<A: Float + Send + Sync> {
    feature_distributions: HashMap<usize, FeatureDistribution<A>>,
    kl_divergence_threshold: A,
    wasserstein_distance_threshold: A,
    distribution_drift_score: A,
}

#[derive(Debug, Clone)]
pub struct FeatureDistribution<A: Float + Send + Sync> {
    mean: A,
    variance: A,
    histogram: Vec<A>,
    last_update: Instant,
}

/// Adaptation speed controller for drift response
#[derive(Debug, Clone)]
pub struct AdaptationSpeedController<A: Float + Send + Sync> {
    base_adaptation_rate: A,
    current_adaptation_rate: A,
    acceleration_factor: A,
    deceleration_factor: A,
    momentum: A,
}

/// Drift severity assessment
#[derive(Debug, Clone)]
pub struct DriftSeverityAssessor<A: Float + Send + Sync> {
    severity_levels: Vec<DriftSeverityLevel<A>>,
    current_severity: DriftSeverityLevel<A>,
    severity_history: VecDeque<DriftSeverityLevel<A>>,
}

#[derive(Debug, Clone)]
pub struct DriftSeverityLevel<A: Float + Send + Sync> {
    level: DriftSeverity,
    magnitude: A,
    recommended_lr_adjustment: A,
    adaptation_urgency: A,
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub enum DriftSeverity {
    None,
    Mild,
    Moderate,
    Severe,
    Critical,
}

/// Resource usage tracking components
#[derive(Debug, Clone, Default)]
pub struct MemoryUsageTracker {
    current_usage_mb: f64,
    peak_usage_mb: f64,
    usage_history: VecDeque<f64>,
    memory_pressure: f64,
}

#[derive(Debug, Clone, Default)]
pub struct ComputationTimeTracker {
    step_times: VecDeque<Duration>,
    average_step_time: Duration,
    time_budget: Duration,
    time_pressure: f64,
}

#[derive(Debug, Clone, Default)]
pub struct EnergyConsumptionTracker {
    energy_per_step: VecDeque<f64>,
    cumulative_energy: f64,
    energy_budget: f64,
    energy_efficiency: f64,
}

#[derive(Debug, Clone)]
pub struct ThroughputRequirements<A: Float + Send + Sync> {
    min_samples_per_second: A,
    target_samples_per_second: A,
    current_throughput: A,
    throughput_deficit: A,
}

#[derive(Debug, Clone)]
pub struct ResourceBudgetManager<A: Float + Send + Sync> {
    memory_budget_mb: f64,
    compute_budget_seconds: f64,
    energy_budget_joules: f64,
    budget_utilization: A,
    budget_violations: usize,
}

/// Learning rate predictor neural network
#[derive(Debug, Clone)]
pub struct LearningRatePredictorNetwork<A: Float + Send + Sync> {
    input_features: Vec<FeatureType>,
    hidden_layers: Vec<usize>,
    weights: Vec<Array2<A>>,
    biases: Vec<Array1<A>>,
    prediction_confidence: A,
}

#[derive(Debug, Clone, Copy)]
pub enum FeatureType {
    GradientNorm,
    LossValue,
    LossGradient,
    ParameterNorm,
    UpdateMagnitude,
    LearningProgress,
    ResourceUtilization,
    DataCharacteristics,
}

/// Hyperparameter update record
#[derive(Debug, Clone)]
pub struct HyperparameterUpdate<A: Float + Send + Sync> {
    timestamp: Instant,
    old_lr: A,
    new_lr: A,
    features: Array1<A>,
    reward: A, // Performance improvement
    exploration_bonus: A,
}

/// Exploration strategy for hyperparameter optimization
#[derive(Debug, Clone)]
pub struct ExplorationStrategy<A: Float + Send + Sync> {
    strategy_type: ExplorationStrategyType,
    exploration_rate: A,
    exploitation_rate: A,
    arm_rewards: HashMap<usize, A>,
    arm_counts: HashMap<usize, usize>,
}

#[derive(Debug, Clone, Copy)]
pub enum ExplorationStrategyType {
    EpsilonGreedy,
    UCB1,
    ThompsonSampling,
    LinUCB,
    ContextualBandit,
}

/// Transfer learning for hyperparameter optimization
#[derive(Debug, Clone)]
pub struct TransferLearner<A: Float + Send + Sync> {
    source_task_data: Vec<TaskData<A>>,
    similarity_metrics: Vec<TaskSimilarityMetric<A>>,
    transfer_weights: Array1<A>,
    transfer_confidence: A,
}

#[derive(Debug, Clone)]
pub struct TaskData<A: Float + Send + Sync> {
    task_id: String,
    optimal_lr_sequence: Vec<A>,
    task_features: Array1<A>,
    performance_curve: Vec<A>,
}

#[derive(Debug, Clone)]
pub struct TaskSimilarityMetric<A: Float + Send + Sync> {
    metric_type: SimilarityMetricType,
    similarity_score: A,
    weight: A,
}

#[derive(Debug, Clone, Copy)]
pub enum SimilarityMetricType {
    DatasetSize,
    ModelArchitecture,
    LossFunction,
    DataDistribution,
    OptimizationLandscape,
}

/// Adaptation statistics for monitoring and analysis
#[derive(Debug, Clone, Default)]
pub struct AdaptationStatistics<A: Float + Send + Sync> {
    /// Total number of adaptations
    pub total_adaptations: usize,

    /// Successful adaptations (led to improvement)
    pub successful_adaptations: usize,

    /// Average adaptation frequency
    pub avg_adaptation_frequency: A,

    /// Learning rate volatility
    pub lr_volatility: A,

    /// Signal reliability scores
    pub signal_reliability_scores: HashMap<AdaptationSignalType, A>,

    /// Adaptation effectiveness by signal type
    pub signal_effectiveness: HashMap<AdaptationSignalType, A>,

    /// Resource efficiency improvements
    pub resource_efficiency_gains: A,

    /// Convergence speed improvement
    pub convergence_speed_improvement: A,
}

impl<A: Float + Default + Clone + Send + Sync + Send + Sync> EnhancedAdaptiveLRController<A> {
    /// Create a new enhanced adaptive learning rate controller
    pub fn new(config: AdaptiveLRConfig<A>) -> Result<Self> {
        let adaptation_strategy = MultiSignalAdaptationStrategy::new(&config)?;
        let gradient_adapter = GradientBasedAdapter::new(&config)?;
        let performance_adapter = PerformanceBasedAdapter::new(&config)?;
        let drift_adapter = DriftAwareAdapter::new(&config)?;
        let resource_adapter = ResourceAwareAdapter::new(&config)?;
        let meta_optimizer = MetaOptimizer::new(&config)?;

        Ok(Self {
            current_lr: config.base_lr,
            base_lr: config.base_lr,
            min_lr: config.min_lr,
            max_lr: config.max_lr,
            adaptation_strategy,
            gradient_adapter,
            performance_adapter,
            drift_adapter,
            resource_adapter,
            meta_optimizer,
            adaptation_history: VecDeque::with_capacity(config.history_window_size),
            config,
        })
    }

    /// Update learning rate based on multiple adaptation signals
    pub fn update_learning_rate(
        &mut self,
        gradients: &Array1<A>,
        loss: A,
        metrics: &HashMap<String, A>,
        step: usize,
    ) -> Result<A> {
        // Collect adaptation signals from all components
        let mut signals = Vec::new();

        if self.config.enable_gradient_adaptation {
            if let Ok(signal) = self.gradient_adapter.generate_signal(gradients, step) {
                signals.push(signal);
            }
        }

        if self.config.enable_performance_adaptation {
            if let Ok(signal) = self
                .performance_adapter
                .generate_signal(loss, metrics, step)
            {
                signals.push(signal);
            }
        }

        if self.config.enable_drift_adaptation {
            if let Ok(signal) = self.drift_adapter.generate_signal(gradients, step) {
                signals.push(signal);
            }
        }

        if self.config.enable_resource_adaptation {
            if let Ok(signal) = self.resource_adapter.generate_signal(step) {
                signals.push(signal);
            }
        }

        // Resolve conflicts and make adaptation decision
        let decision = self.adaptation_strategy.resolve_signals(signals, step)?;

        // Apply meta-learning if enabled
        if self.config.enable_meta_learning {
            let meta_adjustment = self.meta_optimizer.meta_optimize(&decision, step)?;
            self.current_lr = self.apply_meta_adjustment(decision.new_lr, meta_adjustment);
        } else {
            self.current_lr = decision.new_lr;
        }

        // Ensure learning rate is within bounds
        self.current_lr = self
            .current_lr
            .clamp(self.config.min_lr, self.config.max_lr);

        // Record adaptation event
        let event = AdaptationEvent {
            timestamp: Instant::now(),
            old_lr: decision.new_lr, // Store for comparison
            new_lr: self.current_lr,
            trigger_signals: decision.contributing_signals,
            adaptation_reason: decision.rationale,
            confidence: decision.confidence,
            effectiveness_score: None, // Will be updated later
        };

        self.adaptation_history.push_back(event);
        if self.adaptation_history.len() > self.config.history_window_size {
            self.adaptation_history.pop_front();
        }

        Ok(self.current_lr)
    }

    /// Get current learning rate
    pub fn get_current_lr(&self) -> A {
        self.current_lr
    }

    /// Get adaptation statistics
    pub fn get_adaptation_statistics(&self) -> AdaptationStatistics<A> {
        let total_adaptations = self.adaptation_history.len();
        let successful_adaptations = self
            .adaptation_history
            .iter()
            .filter(|event| {
                event
                    .effectiveness_score
                    .is_some_and(|score| score > A::zero())
            })
            .count();

        let lr_volatility = if !self.adaptation_history.is_empty() {
            let lr_values: Vec<A> = self
                .adaptation_history
                .iter()
                .map(|event| event.new_lr)
                .collect();

            let mean_lr = lr_values.iter().fold(A::zero(), |acc, &lr| acc + lr)
                / A::from(lr_values.len()).unwrap();

            let variance = lr_values
                .iter()
                .map(|&lr| {
                    let diff = lr - mean_lr;
                    diff * diff
                })
                .fold(A::zero(), |acc, var| acc + var)
                / A::from(lr_values.len()).unwrap();

            variance.sqrt()
        } else {
            A::zero()
        };

        AdaptationStatistics {
            total_adaptations,
            successful_adaptations,
            lr_volatility,
            ..Default::default()
        }
    }

    /// Apply meta-learning adjustment to base decision
    fn apply_meta_adjustment(&self, base_lr: A, meta_adjustment: A) -> A {
        // Combine base decision with meta-learning recommendation
        let alpha = A::from(0.7).unwrap(); // Weight for base decision
        let beta = A::from(0.3).unwrap(); // Weight for meta-learning

        alpha * base_lr + beta * meta_adjustment
    }

    /// Evaluate adaptation effectiveness retrospectively
    pub fn evaluate_adaptation_effectiveness(&mut self, performance_improvement: A) {
        if let Some(last_event) = self.adaptation_history.back_mut() {
            last_event.effectiveness_score = Some(performance_improvement);

            // Update signal reliability based on effectiveness
            for signal_type in &last_event.trigger_signals {
                self.adaptation_strategy
                    .update_signal_reliability(*signal_type, performance_improvement);
            }
        }
    }

    /// Reset controller state
    pub fn reset(&mut self) {
        self.current_lr = self.base_lr;
        self.adaptation_history.clear();
        self.gradient_adapter.reset();
        self.performance_adapter.reset();
        self.drift_adapter.reset();
        self.resource_adapter.reset();
        self.meta_optimizer.reset();
    }
}

// Implementation stubs for the various components
// In a full implementation, these would contain sophisticated algorithms

impl<A: Float + Default + Clone + Send + Sync + Send + Sync> MultiSignalAdaptationStrategy<A> {
    fn new(config: &AdaptiveLRConfig<A>) -> Result<Self> {
        Ok(Self {
            signal_weights: HashMap::new(),
            voting_history: VecDeque::new(),
            conflict_resolution: ConflictResolution::WeightedAverage,
            signal_reliability: HashMap::new(),
            last_decision: None,
        })
    }

    fn resolve_signals(
        &mut self,
        signals: Vec<SignalVote<A>>,
        _step: usize,
    ) -> Result<AdaptationDecision<A>> {
        if signals.is_empty() {
            return Ok(AdaptationDecision {
                new_lr: A::from(0.001).unwrap(),
                lr_multiplier: A::one(),
                contributing_signals: vec![],
                confidence: A::zero(),
                rationale: "No signals available".to_string(),
                timestamp: Instant::now(),
            });
        }

        // Simplified conflict resolution using weighted average
        let total_weight = signals
            .iter()
            .map(|s| s.confidence)
            .fold(A::zero(), |acc, c| acc + c);

        let weighted_change = signals
            .iter()
            .map(|s| s.recommended_lr_change * s.confidence)
            .fold(A::zero(), |acc, change| acc + change)
            / total_weight;

        let contributing_signals = signals.iter().map(|s| s.signal_type).collect();

        Ok(AdaptationDecision {
            new_lr: A::from(0.001).unwrap() * weighted_change,
            lr_multiplier: weighted_change,
            contributing_signals,
            confidence: total_weight / A::from(signals.len()).unwrap(),
            rationale: "Weighted average of adaptation signals".to_string(),
            timestamp: Instant::now(),
        })
    }

    fn update_signal_reliability(&mut self, signal_type: AdaptationSignalType, effectiveness: A) {
        let reliability = self
            .signal_reliability
            .entry(signal_type)
            .or_insert(A::from(0.5).unwrap());

        // Update reliability using exponential moving average
        let alpha = A::from(0.1).unwrap();
        *reliability = (*reliability) * (A::one() - alpha) + effectiveness * alpha;
    }
}

impl<A: Float + Default + Clone + Send + Sync + Send + Sync> GradientBasedAdapter<A> {
    fn new(config: &AdaptiveLRConfig<A>) -> Result<Self> {
        Ok(Self {
            magnitude_history: VecDeque::new(),
            direction_variance_history: VecDeque::new(),
            norm_statistics: GradientNormStatistics::default(),
            snr_estimator: SignalToNoiseEstimator::default(),
            staleness_detector: GradientStalenessDetector::default(),
        })
    }

    fn generate_signal(&mut self, gradients: &Array1<A>, step: usize) -> Result<SignalVote<A>> {
        let magnitude = gradients
            .iter()
            .map(|&g| g * g)
            .fold(A::zero(), |acc, x| acc + x)
            .sqrt();
        self.magnitude_history.push_back(magnitude);

        if self.magnitude_history.len() > 100 {
            self.magnitude_history.pop_front();
        }

        // Simple adaptation based on gradient magnitude
        let recommended_change = if magnitude > A::from(1.0).unwrap() {
            A::from(0.9).unwrap() // Decrease LR for large gradients
        } else if magnitude < A::from(0.01).unwrap() {
            A::from(1.1).unwrap() // Increase LR for small gradients
        } else {
            A::one() // No change
        };

        Ok(SignalVote {
            signal_type: AdaptationSignalType::GradientMagnitude,
            recommended_lr_change: recommended_change,
            confidence: A::from(0.7).unwrap(),
            reasoning: "Gradient magnitude-based adaptation".to_string(),
            timestamp: Instant::now(),
        })
    }

    fn reset(&mut self) {
        self.magnitude_history.clear();
        self.direction_variance_history.clear();
    }
}

impl<A: Float + Default + Clone + Send + Sync + Send + Sync> PerformanceBasedAdapter<A> {
    fn new(config: &AdaptiveLRConfig<A>) -> Result<Self> {
        Ok(Self {
            metric_history: HashMap::new(),
            trend_analyzer: PerformanceTrendAnalyzer::default(),
            plateau_detector: PlateauDetector::default(),
            overfitting_detector: OverfittingDetector::default(),
            efficiency_tracker: LearningEfficiencyTracker::default(),
        })
    }

    fn generate_signal(
        &mut self,
        loss: A,
        metrics: &HashMap<String, A>,
        _step: usize,
    ) -> Result<SignalVote<A>> {
        let loss_history = self.metric_history.entry("loss".to_string()).or_default();

        loss_history.push_back(loss);
        if loss_history.len() > 50 {
            loss_history.pop_front();
        }

        // Simple trend analysis
        let recommended_change = if loss_history.len() >= 2 {
            let recent_loss = loss_history.back().unwrap();
            let prev_loss = loss_history.get(loss_history.len() - 2).unwrap();

            if *recent_loss > *prev_loss {
                A::from(0.95).unwrap() // Decrease LR if loss increased
            } else {
                A::from(1.02).unwrap() // Slight increase if loss decreased
            }
        } else {
            A::one()
        };

        Ok(SignalVote {
            signal_type: AdaptationSignalType::LossProgression,
            recommended_lr_change: recommended_change,
            confidence: A::from(0.8).unwrap(),
            reasoning: "Loss progression analysis".to_string(),
            timestamp: Instant::now(),
        })
    }

    fn reset(&mut self) {
        self.metric_history.clear();
    }
}

impl<A: Float + Default + Clone + Send + Sync + Send + Sync> DriftAwareAdapter<A> {
    fn new(config: &AdaptiveLRConfig<A>) -> Result<Self> {
        Ok(Self {
            drift_detectors: vec![],
            distribution_tracker: DistributionTracker::default(),
            adaptation_speed: AdaptationSpeedController::default(),
            drift_severity: DriftSeverityAssessor::default(),
        })
    }

    fn generate_signal(&mut self, gradients: &Array1<A>, step: usize) -> Result<SignalVote<A>> {
        // Simplified drift detection
        Ok(SignalVote {
            signal_type: AdaptationSignalType::ConceptDrift,
            recommended_lr_change: A::one(),
            confidence: A::from(0.5).unwrap(),
            reasoning: "No drift detected".to_string(),
            timestamp: Instant::now(),
        })
    }

    fn reset(&mut self) {
        // Reset drift detection state
    }
}

impl<A: Float + Default + Clone + Send + Sync + Send + Sync> ResourceAwareAdapter<A> {
    fn new(config: &AdaptiveLRConfig<A>) -> Result<Self> {
        Ok(Self {
            memory_tracker: MemoryUsageTracker::default(),
            compute_tracker: ComputationTimeTracker::default(),
            energy_tracker: EnergyConsumptionTracker::default(),
            throughput_requirements: ThroughputRequirements {
                min_samples_per_second: A::from(100.0).unwrap(),
                target_samples_per_second: A::from(1000.0).unwrap(),
                current_throughput: A::from(500.0).unwrap(),
                throughput_deficit: A::zero(),
            },
            budget_manager: ResourceBudgetManager {
                memory_budget_mb: 1000.0,
                compute_budget_seconds: 3600.0,
                energy_budget_joules: 1000.0,
                budget_utilization: A::from(0.5).unwrap(),
                budget_violations: 0,
            },
        })
    }

    fn generate_signal(&mut self, step: usize) -> Result<SignalVote<A>> {
        // Simplified resource-based adaptation
        let memory_pressure = self.memory_tracker.memory_pressure;

        let recommended_change = if memory_pressure > 0.8 {
            A::from(0.9).unwrap() // Reduce LR to decrease memory usage
        } else if memory_pressure < 0.3 {
            A::from(1.05).unwrap() // Can afford to increase LR
        } else {
            A::one()
        };

        Ok(SignalVote {
            signal_type: AdaptationSignalType::ResourceUtilization,
            recommended_lr_change: recommended_change,
            confidence: A::from(0.6).unwrap(),
            reasoning: format!("Memory pressure: {:.2}", memory_pressure),
            timestamp: Instant::now(),
        })
    }

    fn reset(&mut self) {
        self.memory_tracker = MemoryUsageTracker::default();
        self.compute_tracker = ComputationTimeTracker::default();
        self.energy_tracker = EnergyConsumptionTracker::default();
    }
}

impl<A: Float + Default + Clone + Send + Sync + Send + Sync> MetaOptimizer<A> {
    fn new(config: &AdaptiveLRConfig<A>) -> Result<Self> {
        Ok(Self {
            lr_predictor: LearningRatePredictorNetwork::default(),
            optimization_history: VecDeque::new(),
            exploration_strategy: ExplorationStrategy::default(),
            transfer_learner: TransferLearner::default(),
        })
    }

    fn meta_optimize(&mut self, decision: &AdaptationDecision<A>, step: usize) -> Result<A> {
        // Simplified meta-optimization
        Ok(A::from(0.001).unwrap())
    }

    fn reset(&mut self) {
        self.optimization_history.clear();
    }
}

// Default implementations for various structures
impl<A: Float + Default + Send + Sync + Send + Sync> Default for GradientNormStatistics<A> {
    fn default() -> Self {
        Self {
            mean: A::default(),
            variance: A::default(),
            skewness: A::default(),
            kurtosis: A::default(),
            percentiles: vec![A::default(); 5],
            autocorrelation: A::default(),
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for SignalToNoiseEstimator<A> {
    fn default() -> Self {
        Self {
            signal_estimate: A::default(),
            noise_estimate: A::default(),
            snr_history: VecDeque::new(),
            estimation_method: SNREstimationMethod::MovingAverage,
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for GradientStalenessDetector<A> {
    fn default() -> Self {
        Self {
            staleness_threshold: Duration::from_secs(1),
            gradient_timestamps: VecDeque::new(),
            staleness_impact_model: StalenessImpactModel::default(),
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for StalenessImpactModel<A> {
    fn default() -> Self {
        Self {
            staleness_penalty: A::default(),
            compensation_factor: A::default(),
            impact_history: VecDeque::new(),
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for PerformanceTrendAnalyzer<A> {
    fn default() -> Self {
        Self {
            trend_detection_window: 10,
            trend_types: vec![],
            trend_strength: A::default(),
            trend_duration: Duration::from_secs(0),
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for PlateauDetector<A> {
    fn default() -> Self {
        Self {
            plateau_threshold: A::default(),
            min_plateau_duration: 5,
            current_plateau_length: 0,
            plateau_confidence: A::default(),
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for OverfittingDetector<A> {
    fn default() -> Self {
        Self {
            train_loss_history: VecDeque::new(),
            val_loss_history: VecDeque::new(),
            overfitting_threshold: A::default(),
            early_stopping_patience: 10,
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for LearningEfficiencyTracker<A> {
    fn default() -> Self {
        Self {
            loss_reduction_per_step: VecDeque::new(),
            parameter_change_magnitude: VecDeque::new(),
            efficiency_score: A::default(),
            efficiency_trend: TrendType::Improving,
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for DistributionTracker<A> {
    fn default() -> Self {
        Self {
            feature_distributions: HashMap::new(),
            kl_divergence_threshold: A::default(),
            wasserstein_distance_threshold: A::default(),
            distribution_drift_score: A::default(),
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for AdaptationSpeedController<A> {
    fn default() -> Self {
        Self {
            base_adaptation_rate: A::from(0.1).unwrap_or_default(),
            current_adaptation_rate: A::from(0.1).unwrap_or_default(),
            acceleration_factor: A::from(1.1).unwrap_or_default(),
            deceleration_factor: A::from(0.9).unwrap_or_default(),
            momentum: A::default(),
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for DriftSeverityAssessor<A> {
    fn default() -> Self {
        Self {
            severity_levels: vec![],
            current_severity: DriftSeverityLevel::default(),
            severity_history: VecDeque::new(),
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for DriftSeverityLevel<A> {
    fn default() -> Self {
        Self {
            level: DriftSeverity::None,
            magnitude: A::default(),
            recommended_lr_adjustment: A::one(),
            adaptation_urgency: A::default(),
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for LearningRatePredictorNetwork<A> {
    fn default() -> Self {
        Self {
            input_features: vec![],
            hidden_layers: vec![],
            weights: vec![],
            biases: vec![],
            prediction_confidence: A::default(),
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for ExplorationStrategy<A> {
    fn default() -> Self {
        Self {
            strategy_type: ExplorationStrategyType::EpsilonGreedy,
            exploration_rate: A::from(0.1).unwrap_or_default(),
            exploitation_rate: A::from(0.9).unwrap_or_default(),
            arm_rewards: HashMap::new(),
            arm_counts: HashMap::new(),
        }
    }
}

impl<A: Float + Default + Send + Sync + Send + Sync> Default for TransferLearner<A> {
    fn default() -> Self {
        Self {
            source_task_data: vec![],
            similarity_metrics: vec![],
            transfer_weights: Array1::from_vec(vec![]),
            transfer_confidence: A::default(),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scirs2_core::ndarray::Array1;

    #[test]
    fn test_enhanced_adaptive_lr_controller_creation() {
        let config = AdaptiveLRConfig {
            base_lr: 0.01,
            min_lr: 1e-6,
            max_lr: 1.0,
            enable_gradient_adaptation: true,
            enable_performance_adaptation: true,
            enable_drift_adaptation: false,
            enable_resource_adaptation: false,
            enable_meta_learning: false,
            history_window_size: 100,
            adaptation_frequency: 10,
            adaptation_sensitivity: 0.1,
            use_ensemble_voting: true,
        };

        let controller = EnhancedAdaptiveLRController::<f32>::new(config);
        assert!(controller.is_ok());
    }

    #[test]
    fn test_learning_rate_update() {
        let config = AdaptiveLRConfig {
            base_lr: 0.01,
            min_lr: 1e-6,
            max_lr: 1.0,
            enable_gradient_adaptation: true,
            enable_performance_adaptation: true,
            enable_drift_adaptation: false,
            enable_resource_adaptation: false,
            enable_meta_learning: false,
            history_window_size: 100,
            adaptation_frequency: 10,
            adaptation_sensitivity: 0.1,
            use_ensemble_voting: true,
        };

        let mut controller = EnhancedAdaptiveLRController::<f32>::new(config).unwrap();
        let gradients = Array1::from_vec(vec![0.1, 0.2, 0.05]);
        let loss = 0.5;
        let metrics = HashMap::new();

        let new_lr = controller.update_learning_rate(&gradients, loss, &metrics, 1);
        assert!(new_lr.is_ok());
        assert!(new_lr.unwrap() > 0.0);
    }

    #[test]
    fn test_adaptation_statistics() {
        let config = AdaptiveLRConfig {
            base_lr: 0.01,
            min_lr: 1e-6,
            max_lr: 1.0,
            enable_gradient_adaptation: true,
            enable_performance_adaptation: true,
            enable_drift_adaptation: false,
            enable_resource_adaptation: false,
            enable_meta_learning: false,
            history_window_size: 100,
            adaptation_frequency: 10,
            adaptation_sensitivity: 0.1,
            use_ensemble_voting: true,
        };

        let controller = EnhancedAdaptiveLRController::<f32>::new(config).unwrap();
        let stats = controller.get_adaptation_statistics();

        assert_eq!(stats.total_adaptations, 0);
        assert_eq!(stats.successful_adaptations, 0);
    }
}