trustformers_debug/

dashboard.rs

//! Interactive dashboards for real-time monitoring and analysis
// reason: debug/profiling scaffolding — structs are constructed and their fields/methods
// are retained for the data model, serialization completeness, and future consumers that
// do not yet read every member. Consolidated from many item-level #[allow(dead_code)].
#![allow(dead_code)]

use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, VecDeque};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant, SystemTime};
use uuid::Uuid;

use crate::DebugConfig;

/// Real-time metrics for dashboard display
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DashboardMetrics {
    pub timestamp: SystemTime,
    pub loss: Option<f64>,
    pub accuracy: Option<f64>,
    pub learning_rate: Option<f64>,
    pub memory_usage_mb: f64,
    pub gpu_utilization: Option<f64>,
    pub tokens_per_second: Option<f64>,
    pub gradient_norm: Option<f64>,
    pub epoch: Option<u32>,
    pub step: Option<u64>,
}

/// Training monitor for real-time tracking
#[derive(Debug)]
pub struct TrainingMonitor {
    config: DebugConfig,
    metrics_history: VecDeque<DashboardMetrics>,
    max_history: usize,
    start_time: Instant,
    alert_thresholds: AlertThresholds,
    active_alerts: Vec<TrainingAlert>,
}

/// Alert thresholds for training monitoring
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AlertThresholds {
    pub loss_increase_threshold: f64,
    pub gradient_norm_max: f64,
    pub memory_usage_max_mb: f64,
    pub gpu_utilization_min: f64,
    pub learning_rate_min: f64,
    pub tokens_per_second_min: f64,
}

impl Default for AlertThresholds {
    fn default() -> Self {
        Self {
            loss_increase_threshold: 1.5,
            gradient_norm_max: 10.0,
            memory_usage_max_mb: 8192.0,
            gpu_utilization_min: 0.7,
            learning_rate_min: 1e-8,
            tokens_per_second_min: 100.0,
        }
    }
}

/// Training alert types
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TrainingAlert {
    pub alert_type: AlertType,
    pub severity: AlertSeverity,
    pub message: String,
    pub timestamp: SystemTime,
    pub metric_value: f64,
    pub threshold: f64,
    pub suggested_action: String,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum AlertType {
    LossIncrease,
    GradientExplosion,
    MemoryOveruse,
    LowGpuUtilization,
    LearningRateTooLow,
    SlowTokenProcessing,
    ModelDivergence,
    TrainingStalled,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum AlertSeverity {
    Info,
    Warning,
    Critical,
}

impl TrainingMonitor {
    /// Create a new training monitor
    pub fn new(config: &DebugConfig) -> Self {
        Self {
            config: config.clone(),
            metrics_history: VecDeque::new(),
            max_history: 10000,
            start_time: Instant::now(),
            alert_thresholds: AlertThresholds::default(),
            active_alerts: Vec::new(),
        }
    }

    /// Update metrics and check for alerts
    pub fn update_metrics(&mut self, metrics: DashboardMetrics) {
        // Add to history
        self.metrics_history.push_back(metrics.clone());

        // Trim history if needed
        if self.metrics_history.len() > self.max_history {
            self.metrics_history.pop_front();
        }

        // Check for alerts
        self.check_alerts(&metrics);
    }

    /// Get recent metrics for dashboard
    pub fn get_recent_metrics(&self, count: usize) -> Vec<DashboardMetrics> {
        self.metrics_history.iter().rev().take(count).rev().cloned().collect()
    }

    /// Get active alerts
    pub fn get_active_alerts(&self) -> &[TrainingAlert] {
        &self.active_alerts
    }

    /// Drop every active alert whose [`AlertType`] equals `alert_type`,
    /// leaving alerts of any other type in place.
    ///
    /// The previous body was
    /// `retain(|alert| !matches!(&alert.alert_type, _alert_type))`. In
    /// `matches!` the second operand is a *pattern*, and a bare identifier
    /// there is an irrefutable binding rather than a comparison against the
    /// parameter -- so the arm always matched, the negation was always
    /// `false`, and the call cleared **all** alerts regardless of which type
    /// was asked for. The `_` prefix additionally silenced the
    /// unused-variable lint that would otherwise have exposed it.
    pub fn clear_alert(&mut self, alert_type: AlertType) {
        self.active_alerts.retain(|alert| alert.alert_type != alert_type);
    }

    /// Set custom alert thresholds
    pub fn set_alert_thresholds(&mut self, thresholds: AlertThresholds) {
        self.alert_thresholds = thresholds;
    }

    /// Generate training summary
    pub fn generate_training_summary(&self) -> TrainingSummary {
        let total_duration = self.start_time.elapsed();
        let total_steps = self.metrics_history.len();

        let avg_loss = self.calculate_average_loss();
        let best_accuracy = self.calculate_best_accuracy();
        let avg_tokens_per_second = self.calculate_average_tokens_per_second();
        let training_stability = self.calculate_training_stability();

        TrainingSummary {
            total_duration,
            total_steps,
            avg_loss,
            best_accuracy,
            avg_tokens_per_second,
            training_stability,
            active_alerts_count: self.active_alerts.len(),
            convergence_status: self.assess_convergence(),
        }
    }

    fn check_alerts(&mut self, metrics: &DashboardMetrics) {
        // Check for loss increase
        if let Some(current_loss) = metrics.loss {
            if let Some(prev_metrics) =
                self.metrics_history.get(self.metrics_history.len().saturating_sub(10))
            {
                if let Some(prev_loss) = prev_metrics.loss {
                    if current_loss > prev_loss * self.alert_thresholds.loss_increase_threshold {
                        self.add_alert(TrainingAlert {
                            alert_type: AlertType::LossIncrease,
                            severity: AlertSeverity::Warning,
                            message: "Loss has increased significantly".to_string(),
                            timestamp: SystemTime::now(),
                            metric_value: current_loss,
                            threshold: prev_loss * self.alert_thresholds.loss_increase_threshold,
                            suggested_action: "Check learning rate or data quality".to_string(),
                        });
                    }
                }
            }
        }

        // Check gradient norm
        if let Some(grad_norm) = metrics.gradient_norm {
            if grad_norm > self.alert_thresholds.gradient_norm_max {
                self.add_alert(TrainingAlert {
                    alert_type: AlertType::GradientExplosion,
                    severity: AlertSeverity::Critical,
                    message: "Gradient explosion detected".to_string(),
                    timestamp: SystemTime::now(),
                    metric_value: grad_norm,
                    threshold: self.alert_thresholds.gradient_norm_max,
                    suggested_action: "Apply gradient clipping or reduce learning rate".to_string(),
                });
            }
        }

        // Check memory usage
        if metrics.memory_usage_mb > self.alert_thresholds.memory_usage_max_mb {
            self.add_alert(TrainingAlert {
                alert_type: AlertType::MemoryOveruse,
                severity: AlertSeverity::Warning,
                message: "High memory usage detected".to_string(),
                timestamp: SystemTime::now(),
                metric_value: metrics.memory_usage_mb,
                threshold: self.alert_thresholds.memory_usage_max_mb,
                suggested_action: "Reduce batch size or enable gradient checkpointing".to_string(),
            });
        }

        // Check GPU utilization
        if let Some(gpu_util) = metrics.gpu_utilization {
            if gpu_util < self.alert_thresholds.gpu_utilization_min {
                self.add_alert(TrainingAlert {
                    alert_type: AlertType::LowGpuUtilization,
                    severity: AlertSeverity::Info,
                    message: "Low GPU utilization".to_string(),
                    timestamp: SystemTime::now(),
                    metric_value: gpu_util,
                    threshold: self.alert_thresholds.gpu_utilization_min,
                    suggested_action: "Increase batch size or check data loading".to_string(),
                });
            }
        }

        // Check tokens per second
        if let Some(tps) = metrics.tokens_per_second {
            if tps < self.alert_thresholds.tokens_per_second_min {
                self.add_alert(TrainingAlert {
                    alert_type: AlertType::SlowTokenProcessing,
                    severity: AlertSeverity::Warning,
                    message: "Slow token processing detected".to_string(),
                    timestamp: SystemTime::now(),
                    metric_value: tps,
                    threshold: self.alert_thresholds.tokens_per_second_min,
                    suggested_action: "Optimize model or increase batch size".to_string(),
                });
            }
        }
    }

    fn add_alert(&mut self, alert: TrainingAlert) {
        // Avoid duplicate alerts of same type
        if !self.active_alerts.iter().any(|a| a.alert_type == alert.alert_type) {
            self.active_alerts.push(alert);
        }
    }

    fn calculate_average_loss(&self) -> Option<f64> {
        let losses: Vec<f64> = self.metrics_history.iter().filter_map(|m| m.loss).collect();

        if losses.is_empty() {
            None
        } else {
            Some(losses.iter().sum::<f64>() / losses.len() as f64)
        }
    }

    fn calculate_best_accuracy(&self) -> Option<f64> {
        self.metrics_history
            .iter()
            .filter_map(|m| m.accuracy)
            .fold(None, |acc, x| match acc {
                None => Some(x),
                Some(y) => Some(x.max(y)),
            })
    }

    fn calculate_average_tokens_per_second(&self) -> Option<f64> {
        let tps_values: Vec<f64> =
            self.metrics_history.iter().filter_map(|m| m.tokens_per_second).collect();

        if tps_values.is_empty() {
            None
        } else {
            Some(tps_values.iter().sum::<f64>() / tps_values.len() as f64)
        }
    }

    fn calculate_training_stability(&self) -> TrainingStability {
        if self.metrics_history.len() < 10 {
            return TrainingStability::Insufficient;
        }

        let recent_losses: Vec<f64> =
            self.metrics_history.iter().rev().take(50).filter_map(|m| m.loss).collect();

        if recent_losses.len() < 10 {
            return TrainingStability::Insufficient;
        }

        // Calculate loss variance
        let mean_loss = recent_losses.iter().sum::<f64>() / recent_losses.len() as f64;
        let variance = recent_losses.iter().map(|&x| (x - mean_loss).powi(2)).sum::<f64>()
            / recent_losses.len() as f64;

        let std_dev = variance.sqrt();
        let coefficient_of_variation = if mean_loss != 0.0 { std_dev / mean_loss } else { 0.0 };

        match coefficient_of_variation {
            cv if cv < 0.1 => TrainingStability::Stable,
            cv if cv < 0.3 => TrainingStability::Moderate,
            _ => TrainingStability::Unstable,
        }
    }

    fn assess_convergence(&self) -> ConvergenceStatus {
        if self.metrics_history.len() < 50 {
            return ConvergenceStatus::TooEarly;
        }

        let recent_losses: Vec<f64> =
            self.metrics_history.iter().rev().take(100).filter_map(|m| m.loss).collect();

        if recent_losses.len() < 50 {
            return ConvergenceStatus::TooEarly;
        }

        // Check if loss is decreasing
        let first_half_avg =
            recent_losses[25..].iter().sum::<f64>() / (recent_losses.len() - 25) as f64;
        let second_half_avg = recent_losses[..25].iter().sum::<f64>() / 25.0;

        if second_half_avg < first_half_avg * 0.95 {
            ConvergenceStatus::Converging
        } else if (second_half_avg - first_half_avg).abs() / first_half_avg < 0.01 {
            ConvergenceStatus::Converged
        } else {
            ConvergenceStatus::Diverging
        }
    }
}

/// Model comparison tool for A/B testing
#[derive(Debug)]
pub struct ModelComparator {
    models: HashMap<String, ModelMetrics>,
    comparison_config: ComparisonConfig,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelMetrics {
    pub model_id: String,
    pub model_name: String,
    pub metrics_history: Vec<DashboardMetrics>,
    pub final_loss: Option<f64>,
    pub final_accuracy: Option<f64>,
    pub training_time: Duration,
    pub parameter_count: usize,
    pub model_size_mb: f64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ComparisonConfig {
    pub primary_metric: String,
    pub comparison_window: usize,
    pub significance_threshold: f64,
}

impl Default for ComparisonConfig {
    fn default() -> Self {
        Self {
            primary_metric: "loss".to_string(),
            comparison_window: 100,
            significance_threshold: 0.05,
        }
    }
}

impl ModelComparator {
    /// Create new model comparator
    pub fn new() -> Self {
        Self {
            models: HashMap::new(),
            comparison_config: ComparisonConfig::default(),
        }
    }

    /// Add model for comparison
    pub fn add_model(&mut self, model_metrics: ModelMetrics) {
        self.models.insert(model_metrics.model_id.clone(), model_metrics);
    }

    /// Compare models and generate report
    pub fn compare_models(&self) -> ModelComparisonReport {
        let mut comparisons = Vec::new();
        let model_ids: Vec<String> = self.models.keys().cloned().collect();

        for i in 0..model_ids.len() {
            for j in (i + 1)..model_ids.len() {
                let model_a = &self.models[&model_ids[i]];
                let model_b = &self.models[&model_ids[j]];

                let comparison = self.compare_two_models(model_a, model_b);
                comparisons.push(comparison);
            }
        }

        let best_model = self.find_best_model();
        let ranking = self.rank_models();

        ModelComparisonReport {
            comparisons,
            best_model,
            ranking,
            comparison_config: self.comparison_config.clone(),
        }
    }

    fn compare_two_models(
        &self,
        model_a: &ModelMetrics,
        model_b: &ModelMetrics,
    ) -> ModelComparison {
        let performance_diff = self.calculate_performance_difference(model_a, model_b);
        let efficiency_diff = self.calculate_efficiency_difference(model_a, model_b);
        let statistical_significance = self.test_statistical_significance(model_a, model_b);

        ModelComparison {
            model_a_id: model_a.model_id.clone(),
            model_b_id: model_b.model_id.clone(),
            performance_difference: performance_diff,
            efficiency_difference: efficiency_diff,
            statistical_significance,
            recommendation: self.generate_recommendation(model_a, model_b, performance_diff),
        }
    }

    /// Relative difference in the configured `primary_metric` between the two
    /// models, or `None` when it cannot be computed.
    ///
    /// `None` is returned when either model never recorded the metric, or when
    /// the reference model's value is `0.0` (the relative difference would be
    /// a division by zero). This used to return a bare `0.0` in exactly those
    /// cases, which a caller could not tell apart from the genuine "both
    /// models scored identically" answer.
    fn calculate_performance_difference(
        &self,
        model_a: &ModelMetrics,
        model_b: &ModelMetrics,
    ) -> Option<f64> {
        match self.comparison_config.primary_metric.as_str() {
            "loss" => {
                let (loss_a, loss_b) = (model_a.final_loss?, model_b.final_loss?);
                if loss_a == 0.0 {
                    return None;
                }
                Some((loss_b - loss_a) / loss_a) // Negative means model_a is better
            },
            "accuracy" => {
                let (acc_a, acc_b) = (model_a.final_accuracy?, model_b.final_accuracy?);
                if acc_a == 0.0 {
                    return None;
                }
                Some((acc_b - acc_a) / acc_a) // Positive means model_b is better
            },
            // An unrecognised `primary_metric` names nothing this comparator
            // can read, so there is no difference to report.
            _ => None,
        }
    }

    /// Mean of the relative training-time and model-size differences, or
    /// `None` when either reference quantity is zero.
    ///
    /// A `ModelMetrics` built before training has run (or before the model
    /// size is known) carries `training_time == Duration::ZERO` /
    /// `model_size_mb == 0.0`; dividing by those produced `inf`/`NaN`, and the
    /// only reason the old code did not surface them is that nothing checked.
    /// Absence is now reported as absence.
    fn calculate_efficiency_difference(
        &self,
        model_a: &ModelMetrics,
        model_b: &ModelMetrics,
    ) -> Option<f64> {
        let time_a = model_a.training_time.as_secs_f64();
        if time_a == 0.0 || model_a.model_size_mb == 0.0 {
            return None;
        }

        // Compare training time efficiency
        let time_diff = model_b.training_time.as_secs_f64() / time_a - 1.0;

        // Compare model size efficiency
        let size_diff = model_b.model_size_mb / model_a.model_size_mb - 1.0;

        // Combined efficiency score (lower is better)
        Some((time_diff + size_diff) / 2.0)
    }

    /// Extract the recorded per-step samples of `comparison_config.primary_metric`
    /// ("loss" or "accuracy") from a model's real `metrics_history`, dropping
    /// steps where that metric was not recorded. An unrecognised
    /// `primary_metric` yields no samples (matching
    /// [`Self::calculate_performance_difference`]'s own `_ => None` arm),
    /// never a fabricated series.
    fn metric_samples(&self, model: &ModelMetrics) -> Vec<f64> {
        match self.comparison_config.primary_metric.as_str() {
            "loss" => model.metrics_history.iter().filter_map(|m| m.loss).collect(),
            "accuracy" => model.metrics_history.iter().filter_map(|m| m.accuracy).collect(),
            _ => Vec::new(),
        }
    }

    /// Real two-sample Welch's t-test (unequal variances, unequal sample
    /// sizes) between `model_a` and `model_b`'s recorded per-step
    /// `primary_metric` samples, reusing the same
    /// [`crate::differential_debugging::welch_t_test`] machinery that
    /// backs `DifferentialDebugger::perform_ab_statistical_tests`. Returns
    /// `None` --
    /// never a fabricated `true`/`false` -- when either model has fewer than
    /// 2 recorded samples of the metric, or when the underlying test itself
    /// has no meaningful result (both samples degenerate constants; see
    /// `welch_t_test`'s own doc comment).
    fn test_statistical_significance(
        &self,
        model_a: &ModelMetrics,
        model_b: &ModelMetrics,
    ) -> Option<bool> {
        let samples_a = self.metric_samples(model_a);
        let samples_b = self.metric_samples(model_b);
        crate::differential_debugging::welch_t_test(
            &samples_a,
            &samples_b,
            self.comparison_config.significance_threshold,
        )
        .map(|result| result.is_significant)
    }

    /// Human-readable verdict derived from
    /// [`Self::calculate_performance_difference`]. When that is `None` the
    /// recommendation says so instead of claiming the models are equivalent
    /// (which is what a `0.0` difference used to make it say).
    fn generate_recommendation(
        &self,
        model_a: &ModelMetrics,
        model_b: &ModelMetrics,
        perf_diff: Option<f64>,
    ) -> String {
        let Some(perf_diff) = perf_diff else {
            return format!(
                "Cannot compare {} and {}: the '{}' metric was not recorded for both models",
                model_a.model_name, model_b.model_name, self.comparison_config.primary_metric
            );
        };
        if perf_diff.abs() < 0.01 {
            "Models perform similarly - choose based on other factors".to_string()
        } else if perf_diff < 0.0 {
            format!(
                "Model {} performs {:.1}% better",
                model_a.model_name,
                perf_diff.abs() * 100.0
            )
        } else {
            format!(
                "Model {} performs {:.1}% better",
                model_b.model_name,
                perf_diff * 100.0
            )
        }
    }

    fn find_best_model(&self) -> Option<String> {
        let mut best_model = None;
        let mut best_score = f64::NEG_INFINITY;

        for model in self.models.values() {
            let score = match self.comparison_config.primary_metric.as_str() {
                "loss" => model.final_loss.map(|l| -l).unwrap_or(f64::NEG_INFINITY),
                "accuracy" => model.final_accuracy.unwrap_or(0.0),
                _ => 0.0,
            };

            if score > best_score {
                best_score = score;
                best_model = Some(model.model_id.clone());
            }
        }

        best_model
    }

    fn rank_models(&self) -> Vec<ModelRanking> {
        let mut rankings: Vec<ModelRanking> = self
            .models
            .values()
            .map(|model| {
                let score = match self.comparison_config.primary_metric.as_str() {
                    "loss" => model.final_loss.map(|l| -l).unwrap_or(f64::NEG_INFINITY),
                    "accuracy" => model.final_accuracy.unwrap_or(0.0),
                    _ => 0.0,
                };

                ModelRanking {
                    model_id: model.model_id.clone(),
                    model_name: model.model_name.clone(),
                    score,
                    rank: 0, // Will be filled below
                }
            })
            .collect();

        rankings.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal));

        for (i, ranking) in rankings.iter_mut().enumerate() {
            ranking.rank = i + 1;
        }

        rankings
    }
}

/// Hyperparameter explorer for optimization guidance
#[derive(Debug)]
pub struct HyperparameterExplorer {
    experiments: HashMap<String, HyperparameterExperiment>,
    search_space: HyperparameterSearchSpace,
    optimization_history: Vec<OptimizationStep>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HyperparameterExperiment {
    pub experiment_id: String,
    pub hyperparameters: HashMap<String, HyperparameterValue>,
    pub results: ExperimentResults,
    pub status: ExperimentStatus,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum HyperparameterValue {
    Float(f64),
    Integer(i64),
    String(String),
    Boolean(bool),
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ExperimentResults {
    pub final_loss: Option<f64>,
    pub final_accuracy: Option<f64>,
    pub training_time: Duration,
    pub convergence_epoch: Option<u32>,
    pub best_validation_score: Option<f64>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ExperimentStatus {
    Running,
    Completed,
    Failed,
    Cancelled,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HyperparameterSearchSpace {
    pub learning_rate: (f64, f64),
    pub batch_size: (i64, i64),
    pub dropout_rate: (f64, f64),
    pub weight_decay: (f64, f64),
    pub num_layers: (i64, i64),
    pub hidden_size: (i64, i64),
}

impl Default for HyperparameterSearchSpace {
    fn default() -> Self {
        Self {
            learning_rate: (1e-5, 1e-1),
            batch_size: (4, 128),
            dropout_rate: (0.0, 0.5),
            weight_decay: (0.0, 1e-2),
            num_layers: (1, 12),
            hidden_size: (64, 2048),
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OptimizationStep {
    pub step: usize,
    pub best_experiment_id: String,
    pub best_score: f64,
    pub exploration_count: usize,
    pub exploitation_count: usize,
}

impl HyperparameterExplorer {
    /// Create new hyperparameter explorer
    pub fn new() -> Self {
        Self {
            experiments: HashMap::new(),
            search_space: HyperparameterSearchSpace::default(),
            optimization_history: Vec::new(),
        }
    }

    /// Add experiment result
    pub fn add_experiment(&mut self, experiment: HyperparameterExperiment) {
        self.experiments.insert(experiment.experiment_id.clone(), experiment);
    }

    /// Get hyperparameter recommendations
    pub fn get_recommendations(&self) -> HyperparameterRecommendations {
        let best_experiments = self.find_best_experiments(5);
        let parameter_importance = self.analyze_parameter_importance();
        let suggested_ranges = self.suggest_search_ranges();
        let next_experiments = self.suggest_next_experiments(3);

        HyperparameterRecommendations {
            best_experiments,
            parameter_importance,
            suggested_ranges,
            next_experiments,
            total_experiments: self.experiments.len(),
        }
    }

    fn find_best_experiments(&self, limit: usize) -> Vec<String> {
        let mut experiments: Vec<_> = self.experiments.values().collect();
        experiments.sort_by(|a, b| {
            let score_a = a.results.final_loss.unwrap_or(f64::INFINITY);
            let score_b = b.results.final_loss.unwrap_or(f64::INFINITY);
            score_a.partial_cmp(&score_b).unwrap_or(std::cmp::Ordering::Equal)
        });

        experiments.iter().take(limit).map(|exp| exp.experiment_id.clone()).collect()
    }

    fn analyze_parameter_importance(&self) -> HashMap<String, f64> {
        // Simplified parameter importance analysis
        let mut importance = HashMap::new();
        importance.insert("learning_rate".to_string(), 0.8);
        importance.insert("batch_size".to_string(), 0.6);
        importance.insert("dropout_rate".to_string(), 0.4);
        importance.insert("weight_decay".to_string(), 0.3);
        importance
    }

    fn suggest_search_ranges(&self) -> HashMap<String, (f64, f64)> {
        // Analyze best experiments to narrow search ranges
        let mut ranges = HashMap::new();
        ranges.insert("learning_rate".to_string(), (1e-4, 1e-2));
        ranges.insert("dropout_rate".to_string(), (0.1, 0.3));
        ranges
    }

    fn suggest_next_experiments(&self, count: usize) -> Vec<HashMap<String, HyperparameterValue>> {
        let mut suggestions = Vec::new();

        for i in 0..count {
            let mut params = HashMap::new();

            // Generate varied parameter combinations based on best results
            params.insert(
                "learning_rate".to_string(),
                HyperparameterValue::Float(0.001 * (1.0 + i as f64 * 0.5)),
            );
            params.insert(
                "batch_size".to_string(),
                HyperparameterValue::Integer(32 * (1 + i as i64)),
            );
            params.insert(
                "dropout_rate".to_string(),
                HyperparameterValue::Float(0.1 + i as f64 * 0.1),
            );

            suggestions.push(params);
        }

        suggestions
    }
}

/// Dashboard aggregator that combines all monitoring tools
#[derive(Debug)]
pub struct InteractiveDashboard {
    config: DebugConfig,
    training_monitor: TrainingMonitor,
    model_comparator: ModelComparator,
    hyperparameter_explorer: HyperparameterExplorer,
    dashboard_state: DashboardState,
    websocket_server: Option<DashboardEndpoint>,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct DashboardState {
    pub active_session_id: Option<Uuid>,
    pub refresh_rate_ms: u64,
    pub auto_alerts: bool,
    pub display_mode: DisplayMode,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum DisplayMode {
    Overview,
    DetailedMetrics,
    ModelComparison,
    HyperparameterOptimization,
    AlertsOnly,
}

/// WebSocket server for real-time dashboard updates
#[derive(Debug)]
/// Endpoint configuration plus the queue of dashboard updates waiting to be
/// delivered to clients.
///
/// It is **not** a WebSocket server: nothing binds `port` and no protocol
/// handshake happens here. [`InteractiveDashboard::update`] queues each metrics
/// snapshot; whatever transport the embedding application uses drains the queue
/// with [`InteractiveDashboard::drain_pending_updates`] and delivers them.
pub struct DashboardEndpoint {
    /// Port the embedding application intends to serve on.
    port: u16,
    /// Client identifiers the embedding application has registered.
    connected_clients: Arc<Mutex<Vec<String>>>,
    /// Metrics snapshots queued since the last drain, oldest first, capped at
    /// [`MAX_PENDING_DASHBOARD_UPDATES`].
    pending_updates: Arc<Mutex<VecDeque<DashboardMetrics>>>,
}

impl DashboardEndpoint {
    /// Port the embedding application intends to serve on.
    pub fn port(&self) -> u16 {
        self.port
    }

    /// Client identifiers registered by the embedding application.
    pub fn connected_clients(&self) -> Vec<String> {
        self.connected_clients.lock().map(|clients| clients.clone()).unwrap_or_default()
    }
}

/// Most queued dashboard updates retained before the oldest are dropped.
const MAX_PENDING_DASHBOARD_UPDATES: usize = 1000;

impl InteractiveDashboard {
    /// Create new interactive dashboard
    pub fn new(config: &DebugConfig) -> Self {
        Self {
            config: config.clone(),
            training_monitor: TrainingMonitor::new(config),
            model_comparator: ModelComparator::new(),
            hyperparameter_explorer: HyperparameterExplorer::new(),
            dashboard_state: DashboardState {
                active_session_id: None,
                refresh_rate_ms: 1000,
                auto_alerts: true,
                display_mode: DisplayMode::Overview,
            },
            websocket_server: None,
        }
    }

    /// Start dashboard with WebSocket server
    pub async fn start(&mut self, port: Option<u16>) -> Result<()> {
        let port = port.unwrap_or(8080);

        self.websocket_server = Some(DashboardEndpoint {
            port,
            connected_clients: Arc::new(Mutex::new(Vec::new())),
            pending_updates: Arc::new(Mutex::new(VecDeque::new())),
        });

        // Deliberately not "started on port {port}": no socket is bound here.
        tracing::info!(
            port,
            "interactive dashboard activated; updates will be queued for the embedding \
             application to deliver"
        );
        Ok(())
    }

    /// Take every dashboard update queued since the last call, oldest first.
    ///
    /// Returns an empty vector when the dashboard has not been started.
    pub fn drain_pending_updates(&self) -> Vec<DashboardMetrics> {
        let Some(endpoint) = self.websocket_server.as_ref() else {
            return Vec::new();
        };
        endpoint
            .pending_updates
            .lock()
            .map(|mut queue| queue.drain(..).collect())
            .unwrap_or_default()
    }

    /// Update dashboard with new metrics
    pub fn update(&mut self, metrics: DashboardMetrics) {
        self.training_monitor.update_metrics(metrics.clone());

        // Queue the snapshot for whatever transport the embedding application
        // uses; see `drain_pending_updates`.
        self.queue_update(metrics);
    }

    /// Get current dashboard snapshot
    pub fn get_dashboard_snapshot(&self) -> DashboardSnapshot {
        let training_summary = self.training_monitor.generate_training_summary();
        let recent_metrics = self.training_monitor.get_recent_metrics(100);
        let active_alerts = self.training_monitor.get_active_alerts().to_vec();
        let model_comparison = self.model_comparator.compare_models();
        let hyperparameter_recommendations = self.hyperparameter_explorer.get_recommendations();

        DashboardSnapshot {
            timestamp: SystemTime::now(),
            training_summary,
            recent_metrics,
            active_alerts,
            model_comparison,
            hyperparameter_recommendations,
            dashboard_state: DashboardState {
                active_session_id: self.dashboard_state.active_session_id,
                refresh_rate_ms: self.dashboard_state.refresh_rate_ms,
                auto_alerts: self.dashboard_state.auto_alerts,
                display_mode: self.dashboard_state.display_mode.clone(),
            },
        }
    }

    /// Export dashboard data to file
    pub async fn export_dashboard_data(&self, path: &str) -> Result<()> {
        let snapshot = self.get_dashboard_snapshot();
        let json = serde_json::to_string_pretty(&snapshot)?;
        tokio::fs::write(path, json).await?;
        Ok(())
    }

    /// Queue one metrics snapshot for delivery, dropping the oldest once the
    /// queue reaches [`MAX_PENDING_DASHBOARD_UPDATES`].
    ///
    /// This replaces `broadcast_update`, which sent nothing at all while
    /// logging "Broadcasting dashboard update to connected clients".
    fn queue_update(&self, metrics: DashboardMetrics) {
        let Some(endpoint) = self.websocket_server.as_ref() else {
            return;
        };
        if let Ok(mut queue) = endpoint.pending_updates.lock() {
            queue.push_back(metrics);
            while queue.len() > MAX_PENDING_DASHBOARD_UPDATES {
                queue.pop_front();
            }
        }
    }
}

// Supporting data structures

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TrainingSummary {
    pub total_duration: Duration,
    pub total_steps: usize,
    pub avg_loss: Option<f64>,
    pub best_accuracy: Option<f64>,
    pub avg_tokens_per_second: Option<f64>,
    pub training_stability: TrainingStability,
    pub active_alerts_count: usize,
    pub convergence_status: ConvergenceStatus,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum TrainingStability {
    Stable,
    Moderate,
    Unstable,
    Insufficient,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ConvergenceStatus {
    TooEarly,
    Converging,
    Converged,
    Diverging,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct ModelComparisonReport {
    pub comparisons: Vec<ModelComparison>,
    pub best_model: Option<String>,
    pub ranking: Vec<ModelRanking>,
    pub comparison_config: ComparisonConfig,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct ModelComparison {
    pub model_a_id: String,
    pub model_b_id: String,
    /// Relative change in the configured `primary_metric` from `model_a` to
    /// `model_b`, or `None` when at least one of them never recorded that
    /// metric (or the reference value is zero) -- never a `0.0` standing in
    /// for "nothing was measured".
    pub performance_difference: Option<f64>,
    /// Mean of the relative training-time and model-size changes, or `None`
    /// when `model_a` has no recorded training time or model size.
    pub efficiency_difference: Option<f64>,
    /// `Some(true)`/`Some(false)` from a real Welch's t-test over both
    /// models' recorded `primary_metric` samples (see
    /// `ModelComparator::test_statistical_significance`), or `None` when
    /// there was not enough recorded history to run the test -- never a
    /// fabricated constant.
    pub statistical_significance: Option<bool>,
    pub recommendation: String,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct ModelRanking {
    pub model_id: String,
    pub model_name: String,
    pub score: f64,
    pub rank: usize,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct HyperparameterRecommendations {
    pub best_experiments: Vec<String>,
    pub parameter_importance: HashMap<String, f64>,
    pub suggested_ranges: HashMap<String, (f64, f64)>,
    pub next_experiments: Vec<HashMap<String, HyperparameterValue>>,
    pub total_experiments: usize,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct DashboardSnapshot {
    pub timestamp: SystemTime,
    pub training_summary: TrainingSummary,
    pub recent_metrics: Vec<DashboardMetrics>,
    pub active_alerts: Vec<TrainingAlert>,
    pub model_comparison: ModelComparisonReport,
    pub hyperparameter_recommendations: HyperparameterRecommendations,
    pub dashboard_state: DashboardState,
}

/// Dashboard report for integration with main debug system
#[derive(Debug, Serialize, Deserialize)]
pub struct DashboardReport {
    pub session_duration: Duration,
    pub total_metrics_recorded: usize,
    pub alerts_triggered: usize,
    pub models_compared: usize,
    pub experiments_tracked: usize,
    pub performance_summary: TrainingSummary,
    pub key_insights: Vec<String>,
    pub recommendations: Vec<String>,
}

impl InteractiveDashboard {
    /// Generate comprehensive dashboard report
    pub async fn generate_report(&self) -> Result<DashboardReport> {
        let training_summary = self.training_monitor.generate_training_summary();
        let total_metrics = self.training_monitor.metrics_history.len();
        let alerts_count = self.training_monitor.active_alerts.len();
        let models_count = self.model_comparator.models.len();
        let experiments_count = self.hyperparameter_explorer.experiments.len();

        let key_insights = self.generate_key_insights();
        let recommendations = self.generate_recommendations();

        Ok(DashboardReport {
            session_duration: training_summary.total_duration,
            total_metrics_recorded: total_metrics,
            alerts_triggered: alerts_count,
            models_compared: models_count,
            experiments_tracked: experiments_count,
            performance_summary: training_summary,
            key_insights,
            recommendations,
        })
    }

    fn generate_key_insights(&self) -> Vec<String> {
        let mut insights = Vec::new();

        // Training stability insights
        match self.training_monitor.generate_training_summary().training_stability {
            TrainingStability::Stable => insights.push("Training is proceeding stably".to_string()),
            TrainingStability::Unstable => insights.push(
                "Training shows high variance - consider adjusting hyperparameters".to_string(),
            ),
            _ => {},
        }

        // Model comparison insights
        if self.model_comparator.models.len() > 1 {
            let comparison = self.model_comparator.compare_models();
            if let Some(best_model) = comparison.best_model {
                insights.push(format!("Best performing model: {}", best_model));
            }
        }

        // Alert insights
        let critical_alerts = self
            .training_monitor
            .active_alerts
            .iter()
            .filter(|alert| matches!(alert.severity, AlertSeverity::Critical))
            .count();

        if critical_alerts > 0 {
            insights.push(format!(
                "{} critical alerts require immediate attention",
                critical_alerts
            ));
        }

        insights
    }

    fn generate_recommendations(&self) -> Vec<String> {
        let mut recommendations = Vec::new();

        // Based on active alerts
        for alert in &self.training_monitor.active_alerts {
            if matches!(alert.severity, AlertSeverity::Critical) {
                recommendations.push(alert.suggested_action.clone());
            }
        }

        // Based on hyperparameter exploration
        if self.hyperparameter_explorer.experiments.len() > 5 {
            recommendations.push(
                "Continue hyperparameter optimization with narrowed search ranges".to_string(),
            );
        }

        // Based on model comparison
        if self.model_comparator.models.len() > 1 {
            recommendations
                .push("Focus on the best performing model architecture for production".to_string());
        }

        if recommendations.is_empty() {
            recommendations.push("Continue monitoring training progress".to_string());
        }

        recommendations
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // ---- Wave 6c debug-sweep2: real update queue, no fake broadcast --------

    #[tokio::test]
    async fn dashboard_updates_are_really_queued_for_delivery() {
        let config = DebugConfig::default();
        let mut dashboard = InteractiveDashboard::new(&config);
        // Nothing is queued before the dashboard is started.
        dashboard.update(make_metrics_simple());
        assert!(dashboard.drain_pending_updates().is_empty());

        dashboard.start(Some(9_999)).await.expect("start");
        dashboard.update(make_metrics_simple());
        dashboard.update(make_metrics_simple());

        let drained = dashboard.drain_pending_updates();
        assert_eq!(drained.len(), 2, "both updates must really be retained");
        assert!(
            dashboard.drain_pending_updates().is_empty(),
            "draining must consume the queue"
        );
    }

    #[tokio::test]
    async fn the_pending_update_queue_is_bounded() {
        let config = DebugConfig::default();
        let mut dashboard = InteractiveDashboard::new(&config);
        dashboard.start(None).await.expect("start");
        for _ in 0..(MAX_PENDING_DASHBOARD_UPDATES + 50) {
            dashboard.update(make_metrics_simple());
        }
        assert_eq!(
            dashboard.drain_pending_updates().len(),
            MAX_PENDING_DASHBOARD_UPDATES,
            "the oldest updates must be dropped, not accumulated without bound"
        );
    }

    fn make_config() -> DebugConfig {
        DebugConfig::default()
    }

    fn make_metrics_with(
        loss: Option<f64>,
        accuracy: Option<f64>,
        memory_mb: f64,
    ) -> DashboardMetrics {
        DashboardMetrics {
            timestamp: SystemTime::now(),
            loss,
            accuracy,
            learning_rate: Some(0.001),
            memory_usage_mb: memory_mb,
            gpu_utilization: Some(0.8),
            tokens_per_second: Some(200.0),
            gradient_norm: Some(1.0),
            epoch: Some(1),
            step: Some(100),
        }
    }

    fn make_metrics_simple() -> DashboardMetrics {
        make_metrics_with(Some(0.5), Some(0.85), 2048.0)
    }

    // --- AlertThresholds tests ---

    #[test]
    fn test_alert_thresholds_default() {
        let thresholds = AlertThresholds::default();
        assert!((thresholds.loss_increase_threshold - 1.5).abs() < 1e-9);
        assert!((thresholds.gradient_norm_max - 10.0).abs() < 1e-9);
        assert!((thresholds.memory_usage_max_mb - 8192.0).abs() < 1e-9);
    }

    // --- TrainingMonitor tests ---

    #[test]
    fn test_training_monitor_new() {
        let config = make_config();
        let monitor = TrainingMonitor::new(&config);
        assert!(monitor.metrics_history.is_empty());
        assert!(monitor.active_alerts.is_empty());
        assert_eq!(monitor.max_history, 10000);
    }

    #[test]
    fn test_training_monitor_update_metrics() {
        let config = make_config();
        let mut monitor = TrainingMonitor::new(&config);
        monitor.update_metrics(make_metrics_simple());
        assert_eq!(monitor.metrics_history.len(), 1);
    }

    #[test]
    fn test_training_monitor_history_limit() {
        let config = make_config();
        let mut monitor = TrainingMonitor::new(&config);
        monitor.max_history = 5;
        for _ in 0..10 {
            monitor.update_metrics(make_metrics_simple());
        }
        assert_eq!(monitor.metrics_history.len(), 5);
    }

    #[test]
    fn test_training_monitor_get_recent_metrics() {
        let config = make_config();
        let mut monitor = TrainingMonitor::new(&config);
        for _ in 0..5 {
            monitor.update_metrics(make_metrics_simple());
        }
        let recent = monitor.get_recent_metrics(3);
        assert_eq!(recent.len(), 3);
    }

    #[test]
    fn test_training_monitor_get_recent_metrics_more_than_available() {
        let config = make_config();
        let mut monitor = TrainingMonitor::new(&config);
        monitor.update_metrics(make_metrics_simple());
        let recent = monitor.get_recent_metrics(10);
        assert_eq!(recent.len(), 1);
    }

    #[test]
    fn test_training_monitor_set_alert_thresholds() {
        let config = make_config();
        let mut monitor = TrainingMonitor::new(&config);
        let thresholds = AlertThresholds {
            loss_increase_threshold: 2.0,
            gradient_norm_max: 5.0,
            memory_usage_max_mb: 4096.0,
            gpu_utilization_min: 0.5,
            learning_rate_min: 1e-6,
            tokens_per_second_min: 50.0,
        };
        monitor.set_alert_thresholds(thresholds);
        assert!((monitor.alert_thresholds.gradient_norm_max - 5.0).abs() < 1e-9);
    }

    #[test]
    fn test_training_monitor_gradient_explosion_alert() {
        let config = make_config();
        let mut monitor = TrainingMonitor::new(&config);
        let mut metrics = make_metrics_simple();
        metrics.gradient_norm = Some(100.0);
        monitor.update_metrics(metrics);
        assert!(monitor
            .active_alerts
            .iter()
            .any(|a| a.alert_type == AlertType::GradientExplosion));
    }

    #[test]
    fn test_training_monitor_memory_overuse_alert() {
        let config = make_config();
        let mut monitor = TrainingMonitor::new(&config);
        let metrics = make_metrics_with(Some(0.5), Some(0.8), 10000.0);
        monitor.update_metrics(metrics);
        assert!(monitor.active_alerts.iter().any(|a| a.alert_type == AlertType::MemoryOveruse));
    }

    #[test]
    fn test_training_monitor_low_gpu_alert() {
        let config = make_config();
        let mut monitor = TrainingMonitor::new(&config);
        let mut metrics = make_metrics_simple();
        metrics.gpu_utilization = Some(0.1);
        monitor.update_metrics(metrics);
        assert!(monitor
            .active_alerts
            .iter()
            .any(|a| a.alert_type == AlertType::LowGpuUtilization));
    }

    #[test]
    fn test_training_monitor_slow_token_alert() {
        let config = make_config();
        let mut monitor = TrainingMonitor::new(&config);
        let mut metrics = make_metrics_simple();
        metrics.tokens_per_second = Some(10.0);
        monitor.update_metrics(metrics);
        assert!(monitor
            .active_alerts
            .iter()
            .any(|a| a.alert_type == AlertType::SlowTokenProcessing));
    }

    #[test]
    fn test_training_monitor_no_duplicate_alerts() {
        let config = make_config();
        let mut monitor = TrainingMonitor::new(&config);
        let mut metrics = make_metrics_simple();
        metrics.gradient_norm = Some(100.0);
        monitor.update_metrics(metrics.clone());
        monitor.update_metrics(metrics);
        let grad_alerts = monitor
            .active_alerts
            .iter()
            .filter(|a| a.alert_type == AlertType::GradientExplosion)
            .count();
        assert_eq!(grad_alerts, 1);
    }

    #[test]
    fn test_training_monitor_average_loss_none() {
        let config = make_config();
        let monitor = TrainingMonitor::new(&config);
        assert!(monitor.calculate_average_loss().is_none());
    }

    #[test]
    fn test_training_monitor_average_loss() {
        let config = make_config();
        let mut monitor = TrainingMonitor::new(&config);
        monitor.update_metrics(make_metrics_with(Some(1.0), None, 1024.0));
        monitor.update_metrics(make_metrics_with(Some(2.0), None, 1024.0));
        let avg = monitor.calculate_average_loss();
        assert!(avg.is_some());
        assert!((avg.expect("should be some") - 1.5).abs() < 1e-9);
    }

    #[test]
    fn test_training_monitor_best_accuracy_none() {
        let config = make_config();
        let monitor = TrainingMonitor::new(&config);
        assert!(monitor.calculate_best_accuracy().is_none());
    }

    #[test]
    fn test_training_monitor_best_accuracy() {
        let config = make_config();
        let mut monitor = TrainingMonitor::new(&config);
        monitor.update_metrics(make_metrics_with(None, Some(0.7), 1024.0));
        monitor.update_metrics(make_metrics_with(None, Some(0.9), 1024.0));
        monitor.update_metrics(make_metrics_with(None, Some(0.8), 1024.0));
        let best = monitor.calculate_best_accuracy();
        assert!(best.is_some());
        assert!((best.expect("should be some") - 0.9).abs() < 1e-9);
    }

    #[test]
    fn test_training_monitor_avg_tps_none() {
        let config = make_config();
        let monitor = TrainingMonitor::new(&config);
        assert!(monitor.calculate_average_tokens_per_second().is_none());
    }

    #[test]
    fn test_training_stability_insufficient() {
        let config = make_config();
        let monitor = TrainingMonitor::new(&config);
        assert!(matches!(
            monitor.calculate_training_stability(),
            TrainingStability::Insufficient
        ));
    }

    #[test]
    fn test_convergence_too_early() {
        let config = make_config();
        let monitor = TrainingMonitor::new(&config);
        assert!(matches!(
            monitor.assess_convergence(),
            ConvergenceStatus::TooEarly
        ));
    }

    #[test]
    fn test_generate_training_summary() {
        let config = make_config();
        let monitor = TrainingMonitor::new(&config);
        let summary = monitor.generate_training_summary();
        assert_eq!(summary.total_steps, 0);
        assert!(matches!(
            summary.convergence_status,
            ConvergenceStatus::TooEarly
        ));
    }

    // --- ModelComparator tests ---

    #[test]
    fn test_model_comparator_new() {
        let comparator = ModelComparator::new();
        assert!(comparator.models.is_empty());
    }

    #[test]
    fn test_model_comparator_add_model() {
        let mut comparator = ModelComparator::new();
        comparator.add_model(ModelMetrics {
            model_id: "m1".to_string(),
            model_name: "Model A".to_string(),
            metrics_history: Vec::new(),
            final_loss: Some(0.5),
            final_accuracy: Some(0.9),
            training_time: Duration::from_secs(100),
            parameter_count: 1000,
            model_size_mb: 10.0,
        });
        assert_eq!(comparator.models.len(), 1);
    }

    #[test]
    fn test_model_comparator_find_best_model_empty() {
        let comparator = ModelComparator::new();
        assert!(comparator.find_best_model().is_none());
    }

    #[test]
    fn test_model_comparator_find_best_model() {
        let mut comparator = ModelComparator::new();
        comparator.add_model(ModelMetrics {
            model_id: "m1".to_string(),
            model_name: "Model A".to_string(),
            metrics_history: Vec::new(),
            final_loss: Some(0.5),
            final_accuracy: Some(0.9),
            training_time: Duration::from_secs(100),
            parameter_count: 1000,
            model_size_mb: 10.0,
        });
        comparator.add_model(ModelMetrics {
            model_id: "m2".to_string(),
            model_name: "Model B".to_string(),
            metrics_history: Vec::new(),
            final_loss: Some(0.3),
            final_accuracy: Some(0.95),
            training_time: Duration::from_secs(200),
            parameter_count: 2000,
            model_size_mb: 20.0,
        });
        let best = comparator.find_best_model();
        assert!(best.is_some());
        assert_eq!(best.expect("should find best"), "m2");
    }

    #[test]
    fn test_model_comparator_rank_models() {
        let mut comparator = ModelComparator::new();
        comparator.add_model(ModelMetrics {
            model_id: "m1".to_string(),
            model_name: "A".to_string(),
            metrics_history: Vec::new(),
            final_loss: Some(0.5),
            final_accuracy: None,
            training_time: Duration::from_secs(100),
            parameter_count: 1000,
            model_size_mb: 10.0,
        });
        let ranking = comparator.rank_models();
        assert_eq!(ranking.len(), 1);
        assert_eq!(ranking[0].rank, 1);
    }

    #[test]
    fn test_model_comparator_generate_recommendation_similar() {
        let comparator = ModelComparator::new();
        let ma = ModelMetrics {
            model_id: "a".to_string(),
            model_name: "A".to_string(),
            metrics_history: Vec::new(),
            final_loss: Some(0.5),
            final_accuracy: None,
            training_time: Duration::from_secs(100),
            parameter_count: 1000,
            model_size_mb: 10.0,
        };
        let rec = comparator.generate_recommendation(&ma, &ma, Some(0.0));
        assert!(rec.contains("similarly"));
    }

    #[test]
    fn test_model_comparator_recommendation_reports_missing_metric_not_similarity() {
        // `None` means "the primary metric was never recorded", which must not
        // be reported as "the two models perform similarly".
        let comparator = ModelComparator::new();
        let ma = ModelMetrics {
            model_id: "a".to_string(),
            model_name: "A".to_string(),
            metrics_history: Vec::new(),
            final_loss: None,
            final_accuracy: None,
            training_time: Duration::from_secs(100),
            parameter_count: 1000,
            model_size_mb: 10.0,
        };
        let rec = comparator.generate_recommendation(&ma, &ma, None);
        assert!(!rec.contains("similarly"), "got {rec}");
        assert!(rec.contains("not recorded"), "got {rec}");
    }

    #[test]
    fn test_performance_difference_is_none_when_metric_never_recorded() {
        let comparator = ModelComparator::new();
        let unmeasured = ModelMetrics {
            model_id: "a".to_string(),
            model_name: "A".to_string(),
            metrics_history: Vec::new(),
            final_loss: None,
            final_accuracy: None,
            training_time: Duration::from_secs(100),
            parameter_count: 1000,
            model_size_mb: 10.0,
        };
        assert_eq!(
            comparator.calculate_performance_difference(&unmeasured, &unmeasured),
            None,
            "no final loss on either side: absence, not a 0.0 tie"
        );

        let measured_a = model_with_loss_history("a", &[0.5]);
        let measured_b = model_with_loss_history("b", &[0.4]);
        let diff = comparator
            .calculate_performance_difference(&measured_a, &measured_b)
            .expect("both models recorded a final loss");
        assert!(
            (diff - (-0.2)).abs() < 1e-12,
            "(0.4 - 0.5)/0.5 = -0.2, got {diff}"
        );
    }

    #[test]
    fn test_efficiency_difference_is_none_without_a_reference_scale() {
        let comparator = ModelComparator::new();
        let zeroed = ModelMetrics {
            model_id: "a".to_string(),
            model_name: "A".to_string(),
            metrics_history: Vec::new(),
            final_loss: Some(0.5),
            final_accuracy: None,
            training_time: Duration::ZERO,
            parameter_count: 1000,
            model_size_mb: 0.0,
        };
        assert_eq!(
            comparator.calculate_efficiency_difference(&zeroed, &zeroed),
            None,
            "dividing by a zero reference used to yield NaN/inf, not a real ratio"
        );

        let a = model_with_loss_history("a", &[0.5]);
        let mut b = model_with_loss_history("b", &[0.5]);
        b.training_time = Duration::from_secs(150);
        b.model_size_mb = 20.0;
        let diff = comparator
            .calculate_efficiency_difference(&a, &b)
            .expect("both scales are non-zero");
        // time 150/100 - 1 = 0.5, size 20/10 - 1 = 1.0, mean = 0.75
        assert!((diff - 0.75).abs() < 1e-12, "got {diff}");
    }

    #[test]
    fn test_clear_alert_removes_only_the_requested_alert_type() {
        let config = DebugConfig::default();
        let mut monitor = TrainingMonitor::new(&config);
        for alert_type in [
            AlertType::LossIncrease,
            AlertType::MemoryOveruse,
            AlertType::TrainingStalled,
        ] {
            monitor.active_alerts.push(TrainingAlert {
                alert_type,
                severity: AlertSeverity::Warning,
                message: "test".to_string(),
                timestamp: SystemTime::now(),
                metric_value: 1.0,
                threshold: 0.5,
                suggested_action: "none".to_string(),
            });
        }
        assert_eq!(monitor.get_active_alerts().len(), 3);

        monitor.clear_alert(AlertType::MemoryOveruse);

        // The old `matches!(x, _alert_type)` body cleared all three.
        let remaining: Vec<&AlertType> =
            monitor.get_active_alerts().iter().map(|a| &a.alert_type).collect();
        assert_eq!(remaining.len(), 2, "only the MemoryOveruse alert may go");
        assert!(remaining.contains(&&AlertType::LossIncrease));
        assert!(remaining.contains(&&AlertType::TrainingStalled));
        assert!(!remaining.contains(&&AlertType::MemoryOveruse));

        // Clearing a type that is not present must be a no-op.
        monitor.clear_alert(AlertType::GradientExplosion);
        assert_eq!(monitor.get_active_alerts().len(), 2);
    }

    fn model_with_loss_history(model_id: &str, losses: &[f64]) -> ModelMetrics {
        ModelMetrics {
            model_id: model_id.to_string(),
            model_name: model_id.to_string(),
            metrics_history: losses
                .iter()
                .map(|&l| make_metrics_with(Some(l), None, 1024.0))
                .collect(),
            final_loss: losses.last().copied(),
            final_accuracy: None,
            training_time: Duration::from_secs(100),
            parameter_count: 1000,
            model_size_mb: 10.0,
        }
    }

    #[test]
    fn test_statistical_significance_none_with_empty_history() {
        // No recorded `loss` samples on either side -- an honest `None`,
        // never a fabricated `true`.
        let comparator = ModelComparator::new();
        let ma = model_with_loss_history("a", &[]);
        let mb = model_with_loss_history("b", &[]);
        assert_eq!(comparator.test_statistical_significance(&ma, &mb), None);
    }

    #[test]
    fn test_statistical_significance_none_with_single_sample() {
        // A single recorded sample per model is not enough for a real
        // two-sample t-test.
        let comparator = ModelComparator::new();
        let ma = model_with_loss_history("a", &[0.5]);
        let mb = model_with_loss_history("b", &[0.2]);
        assert_eq!(comparator.test_statistical_significance(&ma, &mb), None);
    }

    #[test]
    fn test_statistical_significance_true_for_clearly_separated_models() {
        // Two tight, well-separated loss distributions: a real Welch's
        // t-test must find this significant.
        let comparator = ModelComparator::new();
        let ma = model_with_loss_history("a", &[0.50, 0.51, 0.49, 0.50, 0.52, 0.48, 0.50, 0.51]);
        let mb = model_with_loss_history("b", &[0.20, 0.21, 0.19, 0.20, 0.22, 0.18, 0.20, 0.21]);
        assert_eq!(
            comparator.test_statistical_significance(&ma, &mb),
            Some(true)
        );
    }

    #[test]
    fn test_statistical_significance_false_for_overlapping_models() {
        // Two noisy loss distributions with (almost) the same mean and
        // overlapping spread: a real Welch's t-test must NOT find this
        // significant -- this is exactly the case the old `true //
        // Placeholder` got wrong for every pair.
        let comparator = ModelComparator::new();
        let ma = model_with_loss_history("a", &[0.50, 0.55, 0.45, 0.52, 0.48, 0.51, 0.49, 0.53]);
        let mb = model_with_loss_history("b", &[0.51, 0.46, 0.54, 0.49, 0.52, 0.47, 0.53, 0.50]);
        assert_eq!(
            comparator.test_statistical_significance(&ma, &mb),
            Some(false)
        );
    }

    #[test]
    fn test_compare_two_models_publishes_option_significance() {
        // End-to-end: `compare_two_models` (called from `compare_models`,
        // in turn from `get_dashboard_snapshot`) must publish the same
        // real `Option<bool>`, not a constant.
        let comparator = ModelComparator::new();
        let ma = model_with_loss_history("a", &[0.50, 0.51, 0.49, 0.50, 0.52, 0.48, 0.50, 0.51]);
        let mb = model_with_loss_history("b", &[0.20, 0.21, 0.19, 0.20, 0.22, 0.18, 0.20, 0.21]);
        let comparison = comparator.compare_two_models(&ma, &mb);
        assert_eq!(comparison.statistical_significance, Some(true));
    }

    // --- HyperparameterExplorer tests ---

    #[test]
    fn test_hyperparameter_explorer_new() {
        let explorer = HyperparameterExplorer::new();
        assert!(explorer.experiments.is_empty());
    }

    #[test]
    fn test_hyperparameter_explorer_add_experiment() {
        let mut explorer = HyperparameterExplorer::new();
        explorer.add_experiment(HyperparameterExperiment {
            experiment_id: "exp1".to_string(),
            hyperparameters: HashMap::new(),
            results: ExperimentResults {
                final_loss: Some(0.5),
                final_accuracy: Some(0.9),
                training_time: Duration::from_secs(100),
                convergence_epoch: Some(50),
                best_validation_score: Some(0.88),
            },
            status: ExperimentStatus::Completed,
        });
        assert_eq!(explorer.experiments.len(), 1);
    }

    #[test]
    fn test_hyperparameter_explorer_get_recommendations() {
        let explorer = HyperparameterExplorer::new();
        let recs = explorer.get_recommendations();
        assert_eq!(recs.total_experiments, 0);
        assert!(!recs.parameter_importance.is_empty());
    }

    #[test]
    fn test_hyperparameter_explorer_suggest_next_experiments() {
        let explorer = HyperparameterExplorer::new();
        let suggestions = explorer.suggest_next_experiments(3);
        assert_eq!(suggestions.len(), 3);
    }

    // --- InteractiveDashboard tests ---

    #[test]
    fn test_interactive_dashboard_new() {
        let config = make_config();
        let dashboard = InteractiveDashboard::new(&config);
        assert!(dashboard.websocket_server.is_none());
    }

    #[test]
    fn test_interactive_dashboard_update() {
        let config = make_config();
        let mut dashboard = InteractiveDashboard::new(&config);
        dashboard.update(make_metrics_simple());
        assert_eq!(dashboard.training_monitor.metrics_history.len(), 1);
    }

    #[test]
    fn test_interactive_dashboard_snapshot() {
        let config = make_config();
        let dashboard = InteractiveDashboard::new(&config);
        let snapshot = dashboard.get_dashboard_snapshot();
        assert!(snapshot.recent_metrics.is_empty());
    }

    #[test]
    fn test_interactive_dashboard_generate_recommendations() {
        let config = make_config();
        let dashboard = InteractiveDashboard::new(&config);
        let recs = dashboard.generate_recommendations();
        assert!(!recs.is_empty());
    }

    #[test]
    fn test_interactive_dashboard_generate_key_insights() {
        let config = make_config();
        let dashboard = InteractiveDashboard::new(&config);
        let insights = dashboard.generate_key_insights();
        // With no data, stability is insufficient, so minimal insights
        assert!(insights.is_empty() || !insights.is_empty());
    }

    // --- ComparisonConfig tests ---

    #[test]
    fn test_comparison_config_default() {
        let config = ComparisonConfig::default();
        assert_eq!(config.primary_metric, "loss");
        assert_eq!(config.comparison_window, 100);
    }

    // --- HyperparameterSearchSpace tests ---

    #[test]
    fn test_search_space_default() {
        let space = HyperparameterSearchSpace::default();
        assert!(space.learning_rate.0 < space.learning_rate.1);
        assert!(space.batch_size.0 < space.batch_size.1);
    }
}