optirs-learned 0.3.1

use std::fmt::Debug;
// Evaluation metrics and methods for transformer optimization
//
// This module implements comprehensive evaluation strategies for assessing
// the performance of transformer-based learned optimizers across various metrics.

#[allow(dead_code)]
use scirs2_core::ndarray::{Array1, Array2, Array3};
use scirs2_core::numeric::Float;
use std::collections::{HashMap, VecDeque};

use crate::error::{OptimError, Result};

/// Evaluation strategies for transformer optimizers
#[derive(Debug, Clone, Copy)]
pub enum EvaluationStrategy {
    /// Single-task evaluation
    SingleTask,
    /// Multi-task evaluation
    MultiTask,
    /// Cross-domain evaluation
    CrossDomain,
    /// Few-shot evaluation
    FewShot,
    /// Continual learning evaluation
    ContinualLearning,
    /// Robustness evaluation
    Robustness,
    /// Efficiency evaluation
    Efficiency,
    /// Comprehensive evaluation
    Comprehensive,
}

/// Performance evaluator for transformer optimizers
#[derive(Debug, Clone)]
pub struct TransformerEvaluator<T: Float + Debug + Send + Sync + 'static> {
    /// Evaluation strategy
    strategy: EvaluationStrategy,

    /// Evaluation parameters
    eval_params: EvaluationParams<T>,

    /// Metric calculators
    metric_calculators: HashMap<String, MetricCalculator<T>>,

    /// Performance history
    performance_history: VecDeque<EvaluationResult<T>>,

    /// Baseline comparisons
    baseline_comparisons: HashMap<String, BaselineComparison<T>>,

    /// Statistical analyzers
    statistical_analyzers: Vec<StatisticalAnalyzer<T>>,
}

/// Evaluation parameters
#[derive(Debug, Clone)]
pub struct EvaluationParams<T: Float + Debug + Send + Sync + 'static> {
    /// Number of evaluation episodes
    num_episodes: usize,

    /// Evaluation frequency
    eval_frequency: usize,

    /// Convergence tolerance
    convergence_tolerance: T,

    /// Maximum evaluation steps
    max_eval_steps: usize,

    /// Confidence level for statistical tests
    confidence_level: T,

    /// Number of bootstrap samples
    bootstrap_samples: usize,

    /// Cross-validation folds
    cv_folds: usize,

    /// Robustness test severity
    robustness_severity: T,
}

/// Evaluation result
#[derive(Debug, Clone)]
pub struct EvaluationResult<T: Float + Debug + Send + Sync + 'static> {
    /// Evaluation identifier
    eval_id: String,

    /// Task identifier
    task_id: String,

    /// Performance metrics
    metrics: HashMap<String, T>,

    /// Convergence information
    convergence_info: ConvergenceInfo<T>,

    /// Efficiency metrics
    efficiency_metrics: EfficiencyMetrics<T>,

    /// Statistical significance
    statistical_significance: StatisticalSignificance<T>,

    /// Evaluation timestamp
    timestamp: usize,
}

/// Convergence information
#[derive(Debug, Clone)]
pub struct ConvergenceInfo<T: Float + Debug + Send + Sync + 'static> {
    /// Whether convergence was achieved
    converged: bool,

    /// Number of steps to convergence
    steps_to_convergence: Option<usize>,

    /// Final loss value
    final_loss: T,

    /// Convergence rate
    convergence_rate: T,

    /// Loss trajectory
    loss_trajectory: Vec<T>,

    /// Gradient norms
    gradient_norms: Vec<T>,
}

/// Efficiency metrics
#[derive(Debug, Clone)]
pub struct EfficiencyMetrics<T: Float + Debug + Send + Sync + 'static> {
    /// Wall-clock time
    wall_time: T,

    /// Computational FLOPs
    flops: u64,

    /// Memory usage peak
    peak_memory: u64,

    /// Parameter efficiency
    parameter_efficiency: T,

    /// Sample efficiency
    sample_efficiency: T,

    /// Energy consumption estimate
    energy_consumption: T,
}

/// Statistical significance analysis
#[derive(Debug, Clone)]
pub struct StatisticalSignificance<T: Float + Debug + Send + Sync + 'static> {
    /// P-value for performance comparison
    p_value: T,

    /// Effect size
    effect_size: T,

    /// Confidence interval
    confidence_interval: (T, T),

    /// Statistical power
    statistical_power: T,

    /// Test statistic
    test_statistic: T,
}

/// Metric calculator for specific metrics
#[derive(Debug, Clone)]
pub struct MetricCalculator<T: Float + Debug + Send + Sync + 'static> {
    /// Metric name
    metric_name: String,

    /// Calculation function parameters
    calculation_params: HashMap<String, T>,

    /// Historical values for trend analysis
    historical_values: VecDeque<T>,

    /// Aggregation method
    aggregation_method: AggregationMethod,
}

/// Baseline comparison data
#[derive(Debug, Clone)]
pub struct BaselineComparison<T: Float + Debug + Send + Sync + 'static> {
    /// Baseline name
    baseline_name: String,

    /// Baseline performance
    baseline_performance: HashMap<String, T>,

    /// Improvement over baseline
    improvement: HashMap<String, T>,

    /// Relative performance
    relative_performance: HashMap<String, T>,

    /// Win rate against baseline
    win_rate: T,
}

/// Statistical analyzer for performance data
#[derive(Debug, Clone)]
pub struct StatisticalAnalyzer<T: Float + Debug + Send + Sync + 'static> {
    /// Analyzer name
    analyzer_name: String,

    /// Analysis parameters
    analysis_params: HashMap<String, T>,

    /// Results cache
    results_cache: HashMap<String, T>,
}

/// Robustness test suite
#[derive(Debug, Clone)]
pub struct RobustnessTestSuite<T: Float + Debug + Send + Sync + 'static> {
    /// Noise injection tests
    noise_tests: Vec<NoiseTest<T>>,

    /// Adversarial perturbation tests
    adversarial_tests: Vec<AdversarialTest<T>>,

    /// Hyperparameter sensitivity tests
    sensitivity_tests: Vec<SensitivityTest<T>>,

    /// Distribution shift tests
    distribution_tests: Vec<DistributionTest<T>>,
}

/// Individual robustness tests
#[derive(Debug, Clone)]
pub struct NoiseTest<T: Float + Debug + Send + Sync + 'static> {
    noise_type: NoiseType,
    noise_level: T,
    performance_degradation: T,
}

#[derive(Debug, Clone)]
pub struct AdversarialTest<T: Float + Debug + Send + Sync + 'static> {
    attack_type: AttackType,
    attack_strength: T,
    robustness_score: T,
}

#[derive(Debug, Clone)]
pub struct SensitivityTest<T: Float + Debug + Send + Sync + 'static> {
    parameter_name: String,
    parameter_range: (T, T),
    sensitivity_score: T,
}

#[derive(Debug, Clone)]
pub struct DistributionTest<T: Float + Debug + Send + Sync + 'static> {
    shift_type: DistributionShiftType,
    shift_magnitude: T,
    adaptation_score: T,
}

/// Aggregation methods for metrics
#[derive(Debug, Clone, Copy)]
pub enum AggregationMethod {
    Mean,
    Median,
    Max,
    Min,
    WeightedAverage,
    ExponentialMovingAverage,
    Percentile(u8),
}

/// Noise types for robustness testing
#[derive(Debug, Clone, Copy)]
pub enum NoiseType {
    Gaussian,
    Uniform,
    SaltPepper,
    Dropout,
}

/// Attack types for adversarial testing
#[derive(Debug, Clone, Copy)]
pub enum AttackType {
    FGSM,
    PGD,
    CarliniWagner,
    DeepFool,
}

/// Distribution shift types
#[derive(Debug, Clone, Copy)]
pub enum DistributionShiftType {
    CovariateShift,
    ConceptDrift,
    DatasetShift,
    TemporalShift,
}

impl<T: Float + Debug + Default + Clone + Send + Sync + 'static> TransformerEvaluator<T> {
    /// Create new transformer evaluator
    pub fn new(strategy: EvaluationStrategy) -> Result<Self> {
        let mut metric_calculators = HashMap::new();

        // Initialize standard metric calculators
        metric_calculators.insert(
            "convergence_speed".to_string(),
            MetricCalculator::new("convergence_speed".to_string(), AggregationMethod::Mean)?,
        );
        metric_calculators.insert(
            "final_performance".to_string(),
            MetricCalculator::new("final_performance".to_string(), AggregationMethod::Mean)?,
        );
        metric_calculators.insert(
            "sample_efficiency".to_string(),
            MetricCalculator::new("sample_efficiency".to_string(), AggregationMethod::Mean)?,
        );

        Ok(Self {
            strategy,
            eval_params: EvaluationParams::default(),
            metric_calculators,
            performance_history: VecDeque::new(),
            baseline_comparisons: HashMap::new(),
            statistical_analyzers: Vec::new(),
        })
    }

    /// Evaluate transformer optimizer performance
    pub fn evaluate(
        &mut self,
        task_id: &str,
        loss_trajectory: &[T],
        gradient_norms: &[T],
        wall_time: T,
        memory_usage: u64,
    ) -> Result<EvaluationResult<T>> {
        let eval_id = format!("eval_{}_{}", task_id, self.performance_history.len());

        // Compute convergence information
        let convergence_info = self.compute_convergence_info(loss_trajectory, gradient_norms)?;

        // Compute efficiency metrics
        let efficiency_metrics =
            self.compute_efficiency_metrics(wall_time, memory_usage, loss_trajectory.len())?;

        // Compute performance metrics
        let mut metrics = HashMap::new();
        metrics.insert("final_loss".to_string(), convergence_info.final_loss);
        metrics.insert(
            "convergence_rate".to_string(),
            convergence_info.convergence_rate,
        );
        metrics.insert(
            "sample_efficiency".to_string(),
            efficiency_metrics.sample_efficiency,
        );

        // Update metric calculators
        for (metric_name, metric_value) in &metrics {
            if let Some(calculator) = self.metric_calculators.get_mut(metric_name) {
                calculator.update(*metric_value)?;
            }
        }

        // Compute statistical significance if baseline exists
        let statistical_significance = self.compute_statistical_significance(&metrics)?;

        let result = EvaluationResult {
            eval_id,
            task_id: task_id.to_string(),
            metrics,
            convergence_info,
            efficiency_metrics,
            statistical_significance,
            timestamp: self.performance_history.len(),
        };

        self.performance_history.push_back(result.clone());
        if self.performance_history.len() > 1000 {
            self.performance_history.pop_front();
        }

        Ok(result)
    }

    /// Compute convergence information from trajectories
    fn compute_convergence_info(
        &self,
        loss_trajectory: &[T],
        gradient_norms: &[T],
    ) -> Result<ConvergenceInfo<T>> {
        if loss_trajectory.is_empty() {
            return Err(OptimError::InvalidConfig(
                "Empty loss trajectory".to_string(),
            ));
        }

        let final_loss = *loss_trajectory.last().expect("unwrap failed");
        let initial_loss = loss_trajectory[0];

        // Detect convergence
        let (converged, steps_to_convergence) = self.detect_convergence(loss_trajectory)?;

        // Compute convergence rate
        let convergence_rate = if loss_trajectory.len() > 1 {
            let improvement = (initial_loss - final_loss)
                / initial_loss
                    .max(scirs2_core::numeric::NumCast::from(1e-8).unwrap_or_else(|| T::zero()));
            improvement / T::from(loss_trajectory.len() as f64).expect("unwrap failed")
        } else {
            T::zero()
        };

        Ok(ConvergenceInfo {
            converged,
            steps_to_convergence,
            final_loss,
            convergence_rate,
            loss_trajectory: loss_trajectory.to_vec(),
            gradient_norms: gradient_norms.to_vec(),
        })
    }

    /// Detect convergence from loss trajectory
    fn detect_convergence(&self, loss_trajectory: &[T]) -> Result<(bool, Option<usize>)> {
        let window_size = 10.min(loss_trajectory.len());
        let tolerance = self.eval_params.convergence_tolerance;

        if loss_trajectory.len() < window_size {
            return Ok((false, None));
        }

        // Check for convergence: small changes in moving average
        for i in window_size..loss_trajectory.len() {
            let current_window = &loss_trajectory[i - window_size..i];
            let prev_window = &loss_trajectory[i - window_size - 1..i - 1];

            let current_avg = current_window.iter().cloned().fold(T::zero(), |a, b| a + b)
                / scirs2_core::numeric::NumCast::from(window_size as f64)
                    .unwrap_or_else(|| T::zero());
            let prev_avg = prev_window.iter().cloned().fold(T::zero(), |a, b| a + b)
                / scirs2_core::numeric::NumCast::from(window_size as f64)
                    .unwrap_or_else(|| T::zero());

            let change = (current_avg - prev_avg).abs()
                / prev_avg
                    .max(scirs2_core::numeric::NumCast::from(1e-8).unwrap_or_else(|| T::zero()));

            if change < tolerance {
                return Ok((true, Some(i)));
            }
        }

        Ok((false, None))
    }

    /// Compute efficiency metrics
    fn compute_efficiency_metrics(
        &self,
        wall_time: T,
        memory_usage: u64,
        num_steps: usize,
    ) -> Result<EfficiencyMetrics<T>> {
        let flops = (num_steps as u64) * 1000; // Simplified FLOP estimation
        let parameter_efficiency = T::one()
            / (scirs2_core::numeric::NumCast::from(memory_usage as f64)
                .unwrap_or_else(|| T::zero())
                + T::one());
        let sample_efficiency = scirs2_core::numeric::NumCast::from(num_steps as f64)
            .unwrap_or_else(|| T::zero())
            / (wall_time + T::one());
        let energy_consumption = wall_time
            * scirs2_core::numeric::NumCast::from(memory_usage as f64).unwrap_or_else(|| T::zero())
            * scirs2_core::numeric::NumCast::from(1e-9).unwrap_or_else(|| T::zero());

        Ok(EfficiencyMetrics {
            wall_time,
            flops,
            peak_memory: memory_usage,
            parameter_efficiency,
            sample_efficiency,
            energy_consumption,
        })
    }

    /// Compute statistical significance
    fn compute_statistical_significance(
        &self,
        metrics: &HashMap<String, T>,
    ) -> Result<StatisticalSignificance<T>> {
        // Simplified statistical significance computation
        // In practice, this would involve proper statistical tests

        let p_value = scirs2_core::numeric::NumCast::from(0.05).unwrap_or_else(|| T::zero()); // Placeholder
        let effect_size = scirs2_core::numeric::NumCast::from(0.5).unwrap_or_else(|| T::zero()); // Cohen's d
        let confidence_interval = (
            scirs2_core::numeric::NumCast::from(0.1).unwrap_or_else(|| T::zero()),
            scirs2_core::numeric::NumCast::from(0.9).unwrap_or_else(|| T::zero()),
        );
        let statistical_power =
            scirs2_core::numeric::NumCast::from(0.8).unwrap_or_else(|| T::zero());
        let test_statistic = scirs2_core::numeric::NumCast::from(2.0).unwrap_or_else(|| T::zero());

        Ok(StatisticalSignificance {
            p_value,
            effect_size,
            confidence_interval,
            statistical_power,
            test_statistic,
        })
    }

    /// Add baseline for comparison
    pub fn add_baseline(
        &mut self,
        baseline_name: String,
        baseline_performance: HashMap<String, T>,
    ) -> Result<()> {
        let comparison = BaselineComparison {
            baseline_name: baseline_name.clone(),
            baseline_performance: baseline_performance.clone(),
            improvement: HashMap::new(),
            relative_performance: HashMap::new(),
            win_rate: T::zero(),
        };

        self.baseline_comparisons.insert(baseline_name, comparison);
        Ok(())
    }

    /// Run robustness evaluation
    pub fn evaluate_robustness(
        &mut self,
        task_id: &str,
        robustness_tests: &RobustnessTestSuite<T>,
    ) -> Result<HashMap<String, T>> {
        let mut robustness_scores = HashMap::new();

        // Evaluate noise robustness
        let mut noise_score = T::zero();
        for noise_test in &robustness_tests.noise_tests {
            noise_score = noise_score + (T::one() - noise_test.performance_degradation);
        }
        if !robustness_tests.noise_tests.is_empty() {
            noise_score = noise_score
                / T::from(robustness_tests.noise_tests.len() as f64).expect("unwrap failed");
        }
        robustness_scores.insert("noise_robustness".to_string(), noise_score);

        // Evaluate adversarial robustness
        let mut adversarial_score = T::zero();
        for adv_test in &robustness_tests.adversarial_tests {
            adversarial_score = adversarial_score + adv_test.robustness_score;
        }
        if !robustness_tests.adversarial_tests.is_empty() {
            adversarial_score = adversarial_score
                / T::from(robustness_tests.adversarial_tests.len() as f64).expect("unwrap failed");
        }
        robustness_scores.insert("adversarial_robustness".to_string(), adversarial_score);

        // Evaluate hyperparameter sensitivity
        let mut sensitivity_score = T::zero();
        for sens_test in &robustness_tests.sensitivity_tests {
            sensitivity_score =
                sensitivity_score + (T::one() / (T::one() + sens_test.sensitivity_score));
        }
        if !robustness_tests.sensitivity_tests.is_empty() {
            sensitivity_score = sensitivity_score
                / T::from(robustness_tests.sensitivity_tests.len() as f64).expect("unwrap failed");
        }
        robustness_scores.insert("hyperparameter_robustness".to_string(), sensitivity_score);

        Ok(robustness_scores)
    }

    /// Get comprehensive evaluation summary
    pub fn get_evaluation_summary(&self) -> HashMap<String, T> {
        let mut summary = HashMap::new();

        // Overall performance statistics
        if !self.performance_history.is_empty() {
            // Average final performance
            let avg_final_loss = self
                .performance_history
                .iter()
                .map(|result| result.convergence_info.final_loss)
                .fold(T::zero(), |a, b| a + b)
                / T::from(self.performance_history.len() as f64).expect("unwrap failed");
            summary.insert("average_final_loss".to_string(), avg_final_loss);

            // Average convergence rate
            let avg_convergence_rate = self
                .performance_history
                .iter()
                .map(|result| result.convergence_info.convergence_rate)
                .fold(T::zero(), |a, b| a + b)
                / T::from(self.performance_history.len() as f64).expect("unwrap failed");
            summary.insert("average_convergence_rate".to_string(), avg_convergence_rate);

            // Success rate (convergence)
            let success_count = self
                .performance_history
                .iter()
                .filter(|result| result.convergence_info.converged)
                .count();
            let success_rate = scirs2_core::numeric::NumCast::from(success_count as f64)
                .unwrap_or_else(|| T::zero())
                / T::from(self.performance_history.len() as f64).expect("unwrap failed");
            summary.insert("success_rate".to_string(), success_rate);
        }

        summary.insert(
            "total_evaluations".to_string(),
            T::from(self.performance_history.len() as f64).expect("unwrap failed"),
        );
        summary
    }

    /// Reset evaluator state
    pub fn reset(&mut self) {
        self.performance_history.clear();
        self.baseline_comparisons.clear();
        self.statistical_analyzers.clear();

        for calculator in self.metric_calculators.values_mut() {
            calculator.reset();
        }
    }

    /// Update evaluation parameters
    pub fn set_parameters(&mut self, params: EvaluationParams<T>) {
        self.eval_params = params;
    }
}

impl<T: Float + Debug + Default + Clone + Send + Sync + 'static> MetricCalculator<T> {
    fn new(metric_name: String, aggregation_method: AggregationMethod) -> Result<Self> {
        Ok(Self {
            metric_name,
            calculation_params: HashMap::new(),
            historical_values: VecDeque::new(),
            aggregation_method,
        })
    }

    fn update(&mut self, value: T) -> Result<()> {
        self.historical_values.push_back(value);
        if self.historical_values.len() > 1000 {
            self.historical_values.pop_front();
        }
        Ok(())
    }

    fn get_aggregated_value(&self) -> Result<T> {
        if self.historical_values.is_empty() {
            return Ok(T::zero());
        }

        match self.aggregation_method {
            AggregationMethod::Mean => {
                let sum = self
                    .historical_values
                    .iter()
                    .cloned()
                    .fold(T::zero(), |a, b| a + b);
                Ok(sum / T::from(self.historical_values.len() as f64).expect("unwrap failed"))
            }
            AggregationMethod::Max => Ok(self
                .historical_values
                .iter()
                .cloned()
                .fold(T::zero(), |a, b| a.max(b))),
            AggregationMethod::Min => Ok(self.historical_values.iter().cloned().fold(
                scirs2_core::numeric::NumCast::from(f64::INFINITY).unwrap_or_else(|| T::zero()),
                |a, b| a.min(b),
            )),
            _ => Ok(self.historical_values.back().copied().unwrap_or(T::zero())),
        }
    }

    fn reset(&mut self) {
        self.historical_values.clear();
    }
}

impl<T: Float + Debug + Default + Clone + Send + Sync + 'static> Default for EvaluationParams<T> {
    fn default() -> Self {
        Self {
            num_episodes: 10,
            eval_frequency: 100,
            convergence_tolerance: scirs2_core::numeric::NumCast::from(1e-6)
                .unwrap_or_else(|| T::zero()),
            max_eval_steps: 10000,
            confidence_level: scirs2_core::numeric::NumCast::from(0.95)
                .unwrap_or_else(|| T::zero()),
            bootstrap_samples: 1000,
            cv_folds: 5,
            robustness_severity: scirs2_core::numeric::NumCast::from(0.1)
                .unwrap_or_else(|| T::zero()),
        }
    }
}