scirs2_optimize/learned_optimizers/

learned_hyperparameter_tuner.rs

//! Learned Hyperparameter Tuner
//!
//! Implementation of machine learning-based hyperparameter tuning that learns
//! optimal hyperparameter configurations across different optimization problems.

use super::{
    LearnedOptimizationConfig, LearnedOptimizer, MetaOptimizerState, OptimizationProblem,
    TrainingTask,
};
use crate::error::OptimizeResult;
use crate::result::OptimizeResults;
use scirs2_core::ndarray::{Array1, Array2, ArrayView1};
use scirs2_core::random::{Rng, RngExt};
use statrs::statistics::Statistics;
use std::collections::{HashMap, VecDeque};

/// Learned hyperparameter tuner with adaptive configuration
#[derive(Debug, Clone)]
pub struct LearnedHyperparameterTuner {
    /// Configuration
    config: LearnedOptimizationConfig,
    /// Hyperparameter space
    hyperparameter_space: HyperparameterSpace,
    /// Performance database
    performance_database: PerformanceDatabase,
    /// Bayesian optimizer for hyperparameter search
    bayesian_optimizer: BayesianOptimizer,
    /// Multi-fidelity evaluator
    multi_fidelity_evaluator: MultiFidelityEvaluator,
    /// Meta-optimizer state
    meta_state: MetaOptimizerState,
    /// Tuning statistics
    tuning_stats: HyperparameterTuningStats,
}

/// Hyperparameter space definition
#[derive(Debug, Clone)]
pub struct HyperparameterSpace {
    /// Continuous hyperparameters
    continuous_params: Vec<ContinuousHyperparameter>,
    /// Discrete hyperparameters
    discrete_params: Vec<DiscreteHyperparameter>,
    /// Categorical hyperparameters
    categorical_params: Vec<CategoricalHyperparameter>,
    /// Conditional dependencies
    conditional_dependencies: Vec<ConditionalDependency>,
    /// Parameter bounds
    parameter_bounds: HashMap<String, (f64, f64)>,
}

/// Continuous hyperparameter
#[derive(Debug, Clone)]
pub struct ContinuousHyperparameter {
    /// Parameter name
    name: String,
    /// Lower bound
    lower_bound: f64,
    /// Upper bound
    upper_bound: f64,
    /// Scale (linear, log, etc.)
    scale: ParameterScale,
    /// Default value
    default_value: f64,
    /// Importance score
    importance_score: f64,
}

/// Discrete hyperparameter
#[derive(Debug, Clone)]
pub struct DiscreteHyperparameter {
    /// Parameter name
    name: String,
    /// Possible values
    values: Vec<i64>,
    /// Default value
    default_value: i64,
    /// Importance score
    importance_score: f64,
}

/// Categorical hyperparameter
#[derive(Debug, Clone)]
pub struct CategoricalHyperparameter {
    /// Parameter name
    name: String,
    /// Possible categories
    categories: Vec<String>,
    /// Default category
    default_category: String,
    /// Category embeddings
    category_embeddings: HashMap<String, Array1<f64>>,
    /// Importance score
    importance_score: f64,
}

/// Parameter scale types
#[derive(Debug, Clone)]
pub enum ParameterScale {
    Linear,
    Logarithmic,
    Exponential,
    Sigmoid,
}

/// Conditional dependency between parameters
#[derive(Debug, Clone)]
pub struct ConditionalDependency {
    /// Parent parameter
    parent_param: String,
    /// Child parameter
    child_param: String,
    /// Condition
    condition: DependencyCondition,
}

/// Dependency condition types
#[derive(Debug, Clone)]
pub enum DependencyCondition {
    Equals(String),
    GreaterThan(f64),
    LessThan(f64),
    InRange(f64, f64),
    OneOf(Vec<String>),
}

/// Performance database for storing evaluation results
#[derive(Debug, Clone)]
pub struct PerformanceDatabase {
    /// Evaluation records
    records: Vec<EvaluationRecord>,
    /// Indexing for fast retrieval
    index: HashMap<String, Vec<usize>>,
    /// Performance trends
    performance_trends: HashMap<String, PerformanceTrend>,
    /// Correlation matrix
    correlation_matrix: Array2<f64>,
}

/// Evaluation record
#[derive(Debug, Clone)]
pub struct EvaluationRecord {
    /// Hyperparameter configuration
    config: HyperparameterConfig,
    /// Performance metric
    performance: f64,
    /// Evaluation cost
    cost: f64,
    /// Timestamp
    timestamp: u64,
    /// Problem characteristics
    problem_features: Array1<f64>,
    /// Fidelity level
    fidelity: f64,
    /// Additional metrics
    additional_metrics: HashMap<String, f64>,
}

/// Hyperparameter configuration
#[derive(Debug, Clone)]
pub struct HyperparameterConfig {
    /// Parameter values
    parameters: HashMap<String, ParameterValue>,
    /// Configuration hash
    config_hash: u64,
    /// Configuration embedding
    embedding: Array1<f64>,
}

/// Parameter value types
#[derive(Debug, Clone)]
pub enum ParameterValue {
    Continuous(f64),
    Discrete(i64),
    Categorical(String),
}

/// Performance trend analysis
#[derive(Debug, Clone)]
pub struct PerformanceTrend {
    /// Trend direction
    trend_direction: f64,
    /// Trend strength
    trend_strength: f64,
    /// Seasonal patterns
    seasonal_patterns: Array1<f64>,
    /// Volatility measure
    volatility: f64,
}

/// Bayesian optimizer for hyperparameter search
#[derive(Debug, Clone)]
pub struct BayesianOptimizer {
    /// Gaussian process surrogate model
    gaussian_process: GaussianProcess,
    /// Acquisition function
    acquisition_function: AcquisitionFunction,
    /// Optimization strategy
    optimization_strategy: OptimizationStrategy,
    /// Exploration-exploitation balance
    exploration_factor: f64,
}

/// Gaussian process surrogate model
#[derive(Debug, Clone)]
pub struct GaussianProcess {
    /// Training inputs
    training_inputs: Array2<f64>,
    /// Training outputs
    training_outputs: Array1<f64>,
    /// Kernel function
    kernel: KernelFunction,
    /// Kernel hyperparameters
    kernel_params: Array1<f64>,
    /// Noise variance
    noise_variance: f64,
    /// Mean function
    mean_function: MeanFunction,
}

/// Kernel function types
#[derive(Debug, Clone)]
pub enum KernelFunction {
    RBF {
        length_scale: f64,
        variance: f64,
    },
    Matern {
        nu: f64,
        length_scale: f64,
        variance: f64,
    },
    Polynomial {
        degree: i32,
        variance: f64,
    },
    Composite {
        kernels: Vec<KernelFunction>,
        weights: Array1<f64>,
    },
}

/// Mean function for GP
#[derive(Debug, Clone)]
pub enum MeanFunction {
    Zero,
    Constant(f64),
    Linear(Array1<f64>),
    Quadratic(Array2<f64>),
}

/// Acquisition function types
#[derive(Debug, Clone)]
pub enum AcquisitionFunction {
    ExpectedImprovement { xi: f64 },
    ProbabilityOfImprovement { xi: f64 },
    UpperConfidenceBound { beta: f64 },
    EntropySearch { num_samples: usize },
    MultiFidelity { alpha: f64, beta: f64 },
}

/// Optimization strategy for acquisition function
#[derive(Debug, Clone)]
pub enum OptimizationStrategy {
    RandomSearch { num_candidates: usize },
    GridSearch { grid_resolution: usize },
    GradientBased { num_restarts: usize },
    EvolutionarySearch { population_size: usize },
    DIRECT { max_nit: usize },
}

/// Multi-fidelity evaluator
#[derive(Debug, Clone)]
pub struct MultiFidelityEvaluator {
    /// Available fidelity levels
    fidelity_levels: Vec<FidelityLevel>,
    /// Cost model
    cost_model: CostModel,
    /// Fidelity selection strategy
    selection_strategy: FidelitySelectionStrategy,
    /// Correlation estimator
    correlation_estimator: FidelityCorrelationEstimator,
}

/// Fidelity level definition
#[derive(Debug, Clone)]
pub struct FidelityLevel {
    /// Fidelity value (0.0 to 1.0)
    fidelity: f64,
    /// Cost multiplier
    cost_multiplier: f64,
    /// Accuracy estimate
    accuracy: f64,
    /// Resource requirements
    resource_requirements: ResourceRequirements,
}

/// Resource requirements for evaluation
#[derive(Debug, Clone)]
pub struct ResourceRequirements {
    /// Computational time
    computation_time: f64,
    /// Memory usage
    memory_usage: f64,
    /// CPU cores
    cpu_cores: usize,
    /// GPU requirements
    gpu_required: bool,
}

/// Cost model for evaluations
#[derive(Debug, Clone)]
pub struct CostModel {
    /// Cost prediction network
    cost_network: Array2<f64>,
    /// Base cost parameters
    base_cost: f64,
    /// Scaling factors
    scaling_factors: Array1<f64>,
    /// Historical cost data
    cost_history: VecDeque<(f64, f64)>, // (fidelity, cost)
}

/// Fidelity selection strategy
#[derive(Debug, Clone)]
pub enum FidelitySelectionStrategy {
    Static(f64),
    Adaptive {
        initial_fidelity: f64,
        adaptation_rate: f64,
    },
    BanditBased {
        epsilon: f64,
    },
    Predictive {
        prediction_horizon: usize,
    },
}

/// Correlation estimator between fidelities
#[derive(Debug, Clone)]
pub struct FidelityCorrelationEstimator {
    /// Correlation matrix
    correlation_matrix: Array2<f64>,
    /// Estimation method
    estimation_method: CorrelationMethod,
    /// Confidence intervals
    confidence_intervals: Array2<f64>,
}

/// Correlation estimation methods
#[derive(Debug, Clone)]
pub enum CorrelationMethod {
    Pearson,
    Spearman,
    Kendall,
    MutualInformation,
}

/// Hyperparameter tuning statistics
#[derive(Debug, Clone)]
pub struct HyperparameterTuningStats {
    /// Total evaluations performed
    total_evaluations: usize,
    /// Best performance found
    best_performance: f64,
    /// Total cost spent
    total_cost: f64,
    /// Convergence rate
    convergence_rate: f64,
    /// Exploration efficiency
    exploration_efficiency: f64,
    /// Multi-fidelity savings
    multi_fidelity_savings: f64,
}

impl LearnedHyperparameterTuner {
    /// Create new learned hyperparameter tuner
    pub fn new(config: LearnedOptimizationConfig) -> Self {
        let hyperparameter_space = HyperparameterSpace::create_default_space();
        let performance_database = PerformanceDatabase::new();
        let bayesian_optimizer = BayesianOptimizer::new();
        let multi_fidelity_evaluator = MultiFidelityEvaluator::new();
        let hidden_size = config.hidden_size;

        Self {
            config,
            hyperparameter_space,
            performance_database,
            bayesian_optimizer,
            multi_fidelity_evaluator,
            meta_state: MetaOptimizerState {
                meta_params: Array1::zeros(hidden_size),
                network_weights: Array2::zeros((hidden_size, hidden_size)),
                performance_history: Vec::new(),
                adaptation_stats: super::AdaptationStatistics::default(),
                episode: 0,
            },
            tuning_stats: HyperparameterTuningStats::default(),
        }
    }

    /// Tune hyperparameters for optimization problem
    pub fn tune_hyperparameters<F>(
        &mut self,
        objective: F,
        initial_params: &ArrayView1<f64>,
        problem: &OptimizationProblem,
        budget: f64,
    ) -> OptimizeResult<HyperparameterConfig>
    where
        F: Fn(&ArrayView1<f64>) -> f64,
    {
        let mut remaining_budget = budget;
        let mut best_config = self.get_default_config()?;
        let mut best_performance = f64::INFINITY;

        // Extract problem features
        let problem_features =
            self.extract_problem_features(&objective, initial_params, problem)?;

        // Initialize with promising configurations from database
        let promising_configs = self.get_promising_configurations(&problem_features)?;

        // Evaluate promising configurations
        for config in promising_configs {
            if remaining_budget <= 0.0 {
                break;
            }

            let (performance, cost) =
                self.evaluate_configuration(&objective, initial_params, &config)?;
            remaining_budget -= cost;

            // Update database
            self.add_evaluation_record(config.clone(), performance, cost, &problem_features)?;

            if performance < best_performance {
                best_performance = performance;
                best_config = config;
            }
        }

        // Bayesian optimization loop
        while remaining_budget > 0.0 {
            // Update Gaussian process
            self.update_gaussian_process()?;

            // Select next configuration to evaluate
            let next_config = self.select_next_configuration(&problem_features)?;

            // Select fidelity level
            let fidelity = self.select_fidelity_level(&next_config, remaining_budget)?;

            // Evaluate configuration
            let (performance, cost) = self.evaluate_configuration_with_fidelity(
                &objective,
                initial_params,
                &next_config,
                fidelity,
            )?;

            remaining_budget -= cost;

            // Update database
            self.add_evaluation_record(next_config.clone(), performance, cost, &problem_features)?;

            // Update best configuration
            if performance < best_performance {
                best_performance = performance;
                best_config = next_config;
            }

            // Update statistics
            self.update_tuning_stats(performance, cost)?;

            // Check convergence
            if self.check_convergence() {
                break;
            }
        }

        Ok(best_config)
    }

    /// Extract problem features for configuration selection
    fn extract_problem_features<F>(
        &self,
        objective: &F,
        initial_params: &ArrayView1<f64>,
        problem: &OptimizationProblem,
    ) -> OptimizeResult<Array1<f64>>
    where
        F: Fn(&ArrayView1<f64>) -> f64,
    {
        let mut features = Array1::zeros(20);

        // Problem dimension
        features[0] = (problem.dimension as f64).ln();

        // Objective landscape features
        let f0 = objective(initial_params);
        features[1] = f0.abs().ln();

        // Gradient features
        let h = 1e-6;
        let mut gradient_norm = 0.0;
        for i in 0..initial_params.len().min(10) {
            let mut params_plus = initial_params.to_owned();
            params_plus[i] += h;
            let f_plus = objective(&params_plus.view());
            let grad_i = (f_plus - f0) / h;
            gradient_norm += grad_i * grad_i;
        }
        gradient_norm = gradient_norm.sqrt();
        features[2] = gradient_norm.ln();

        // Parameter statistics
        features[3] = initial_params.view().mean();
        features[4] = initial_params.variance().sqrt();
        features[5] = initial_params.fold(-f64::INFINITY, |a, &b| a.max(b));
        features[6] = initial_params.fold(f64::INFINITY, |a, &b| a.min(b));

        // Problem class encoding
        match problem.problem_class.as_str() {
            "quadratic" => features[7] = 1.0,
            "neural_network" => features[8] = 1.0,
            "sparse" => features[9] = 1.0,
            _ => features[10] = 1.0,
        }

        // Budget and accuracy requirements
        features[11] = (problem.max_evaluations as f64).ln();
        features[12] = problem.target_accuracy.ln().abs();

        // Add metadata features
        for (i, (_, &value)) in problem.metadata.iter().enumerate() {
            if 13 + i < features.len() {
                features[13 + i] = value.tanh();
            }
        }

        Ok(features)
    }

    /// Get promising configurations from database
    fn get_promising_configurations(
        &self,
        problem_features: &Array1<f64>,
    ) -> OptimizeResult<Vec<HyperparameterConfig>> {
        let mut configs = Vec::new();
        let mut similarities = Vec::new();

        // Find similar problems in database
        for record in &self.performance_database.records {
            let similarity =
                self.compute_problem_similarity(problem_features, &record.problem_features)?;
            similarities.push((record, similarity));
        }

        // Sort by similarity and performance
        similarities.sort_by(|a, b| {
            let combined_score_a = a.1 * (1.0 / (1.0 + a.0.performance));
            let combined_score_b = b.1 * (1.0 / (1.0 + b.0.performance));
            combined_score_b
                .partial_cmp(&combined_score_a)
                .unwrap_or(std::cmp::Ordering::Equal)
        });

        // Select top configurations
        for (record, similarity) in similarities.into_iter().take(5) {
            configs.push(record.config.clone());
        }

        // Add some random configurations for exploration
        for _ in 0..3 {
            configs.push(self.sample_random_configuration()?);
        }

        Ok(configs)
    }

    /// Compute similarity between problem features
    fn compute_problem_similarity(
        &self,
        features1: &Array1<f64>,
        features2: &Array1<f64>,
    ) -> OptimizeResult<f64> {
        // Cosine similarity
        let dot_product = features1
            .iter()
            .zip(features2.iter())
            .map(|(&a, &b)| a * b)
            .sum::<f64>();

        let norm1 = (features1.iter().map(|&x| x * x).sum::<f64>()).sqrt();
        let norm2 = (features2.iter().map(|&x| x * x).sum::<f64>()).sqrt();

        if norm1 > 0.0 && norm2 > 0.0 {
            Ok(dot_product / (norm1 * norm2))
        } else {
            Ok(0.0)
        }
    }

    /// Sample random configuration from hyperparameter space
    fn sample_random_configuration(&self) -> OptimizeResult<HyperparameterConfig> {
        let mut parameters = HashMap::new();

        // Sample continuous parameters
        for param in &self.hyperparameter_space.continuous_params {
            let value = match param.scale {
                ParameterScale::Linear => {
                    param.lower_bound
                        + scirs2_core::random::rng().random::<f64>()
                            * (param.upper_bound - param.lower_bound)
                }
                ParameterScale::Logarithmic => {
                    let log_lower = param.lower_bound.ln();
                    let log_upper = param.upper_bound.ln();
                    (log_lower
                        + scirs2_core::random::rng().random::<f64>() * (log_upper - log_lower))
                        .exp()
                }
                _ => param.default_value,
            };

            parameters.insert(param.name.clone(), ParameterValue::Continuous(value));
        }

        // Sample discrete parameters
        for param in &self.hyperparameter_space.discrete_params {
            let idx = scirs2_core::random::rng().random_range(0..param.values.len());
            let value = param.values[idx];
            parameters.insert(param.name.clone(), ParameterValue::Discrete(value));
        }

        // Sample categorical parameters
        for param in &self.hyperparameter_space.categorical_params {
            let idx = scirs2_core::random::rng().random_range(0..param.categories.len());
            let value = param.categories[idx].clone();
            parameters.insert(param.name.clone(), ParameterValue::Categorical(value));
        }

        Ok(HyperparameterConfig::new(parameters))
    }

    /// Get default configuration
    fn get_default_config(&self) -> OptimizeResult<HyperparameterConfig> {
        let mut parameters = HashMap::new();

        for param in &self.hyperparameter_space.continuous_params {
            parameters.insert(
                param.name.clone(),
                ParameterValue::Continuous(param.default_value),
            );
        }

        for param in &self.hyperparameter_space.discrete_params {
            parameters.insert(
                param.name.clone(),
                ParameterValue::Discrete(param.default_value),
            );
        }

        for param in &self.hyperparameter_space.categorical_params {
            parameters.insert(
                param.name.clone(),
                ParameterValue::Categorical(param.default_category.clone()),
            );
        }

        Ok(HyperparameterConfig::new(parameters))
    }

    /// Evaluate configuration
    fn evaluate_configuration<F>(
        &self,
        objective: &F,
        initial_params: &ArrayView1<f64>,
        config: &HyperparameterConfig,
    ) -> OptimizeResult<(f64, f64)>
    where
        F: Fn(&ArrayView1<f64>) -> f64,
    {
        self.evaluate_configuration_with_fidelity(objective, initial_params, config, 1.0)
    }

    /// Evaluate configuration with specified fidelity
    fn evaluate_configuration_with_fidelity<F>(
        &self,
        objective: &F,
        initial_params: &ArrayView1<f64>,
        config: &HyperparameterConfig,
        fidelity: f64,
    ) -> OptimizeResult<(f64, f64)>
    where
        F: Fn(&ArrayView1<f64>) -> f64,
    {
        // Create optimizer with specified configuration
        let optimizer_result =
            self.create_optimizer_from_config(config, objective, initial_params, fidelity)?;

        // Compute cost based on fidelity
        let base_cost = 1.0;
        let cost = base_cost * self.multi_fidelity_evaluator.cost_model.base_cost * fidelity;

        Ok((optimizer_result.fun, cost))
    }

    /// Create optimizer from configuration
    fn create_optimizer_from_config<F>(
        &self,
        config: &HyperparameterConfig,
        objective: &F,
        initial_params: &ArrayView1<f64>,
        fidelity: f64,
    ) -> OptimizeResult<OptimizeResults<f64>>
    where
        F: Fn(&ArrayView1<f64>) -> f64,
    {
        // Extract optimization parameters from config
        let learning_rate = match config.parameters.get("learning_rate") {
            Some(ParameterValue::Continuous(lr)) => *lr,
            _ => 0.01,
        };

        let max_nit = match config.parameters.get("max_nit") {
            Some(ParameterValue::Discrete(iters)) => (*iters as f64 * fidelity) as usize,
            _ => (100.0 * fidelity) as usize,
        };

        // Simple optimization with extracted parameters
        let mut current_params = initial_params.to_owned();
        let mut best_value = objective(initial_params);
        let mut best_params = current_params.clone();

        for iter in 0..max_nit {
            // Compute gradient
            let h = 1e-6;
            let f0 = objective(&current_params.view());
            let mut gradient = Array1::zeros(current_params.len());

            for i in 0..current_params.len() {
                let mut params_plus = current_params.clone();
                params_plus[i] += h;
                let f_plus = objective(&params_plus.view());
                gradient[i] = (f_plus - f0) / h;
            }

            // Update parameters
            for i in 0..current_params.len() {
                current_params[i] -= learning_rate * gradient[i];
            }

            let current_value = objective(&current_params.view());
            if current_value < best_value {
                best_value = current_value;
                best_params = current_params.clone();
            }

            // Early stopping for low fidelity
            if fidelity < 1.0 && iter > (max_nit / 2) {
                break;
            }
        }

        Ok(OptimizeResults::<f64> {
            x: best_params,
            fun: best_value,
            success: true,
            nit: max_nit,
            message: "Hyperparameter evaluation completed".to_string(),
            jac: None,
            hess: None,
            constr: None,
            nfev: max_nit,
            njev: 0,
            nhev: 0,
            maxcv: 0,
            status: 0,
        })
    }

    /// Add evaluation record to database
    fn add_evaluation_record(
        &mut self,
        config: HyperparameterConfig,
        performance: f64,
        cost: f64,
        problem_features: &Array1<f64>,
    ) -> OptimizeResult<()> {
        let record = EvaluationRecord {
            config,
            performance,
            cost,
            timestamp: std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)
                .unwrap_or_default()
                .as_secs(),
            problem_features: problem_features.clone(),
            fidelity: 1.0,
            additional_metrics: HashMap::new(),
        };

        self.performance_database.add_record(record);
        Ok(())
    }

    /// Update Gaussian process with new data
    fn update_gaussian_process(&mut self) -> OptimizeResult<()> {
        // Extract training data from database
        let (inputs, outputs) = self.extract_training_data()?;

        // Update GP
        self.bayesian_optimizer
            .gaussian_process
            .update_training_data(inputs, outputs)?;

        // Optimize hyperparameters
        self.bayesian_optimizer
            .gaussian_process
            .optimize_hyperparameters()?;

        Ok(())
    }

    /// Extract training data from database
    fn extract_training_data(&self) -> OptimizeResult<(Array2<f64>, Array1<f64>)> {
        let num_records = self.performance_database.records.len();
        if num_records == 0 {
            return Ok((Array2::zeros((0, 10)), Array1::zeros(0)));
        }

        let input_dim = self.performance_database.records[0].config.embedding.len();
        let mut inputs = Array2::zeros((num_records, input_dim));
        let mut outputs = Array1::zeros(num_records);

        for (i, record) in self.performance_database.records.iter().enumerate() {
            for j in 0..input_dim.min(record.config.embedding.len()) {
                inputs[[i, j]] = record.config.embedding[j];
            }
            outputs[i] = record.performance;
        }

        Ok((inputs, outputs))
    }

    /// Select next configuration to evaluate
    fn select_next_configuration(
        &self,
        _problem_features: &Array1<f64>,
    ) -> OptimizeResult<HyperparameterConfig> {
        // Use acquisition function to select next point
        let candidate_configs = self.generate_candidate_configurations(100)?;
        let mut best_config = candidate_configs[0].clone();
        let mut best_acquisition = f64::NEG_INFINITY;

        for config in candidate_configs {
            let acquisition_value = self.evaluate_acquisition_function(&config)?;
            if acquisition_value > best_acquisition {
                best_acquisition = acquisition_value;
                best_config = config;
            }
        }

        Ok(best_config)
    }

    /// Generate candidate configurations
    fn generate_candidate_configurations(
        &self,
        num_candidates: usize,
    ) -> OptimizeResult<Vec<HyperparameterConfig>> {
        let mut candidates = Vec::new();

        for _ in 0..num_candidates {
            candidates.push(self.sample_random_configuration()?);
        }

        Ok(candidates)
    }

    /// Evaluate acquisition function
    fn evaluate_acquisition_function(&self, config: &HyperparameterConfig) -> OptimizeResult<f64> {
        // Predict mean and variance using GP
        let (mean, variance) = self
            .bayesian_optimizer
            .gaussian_process
            .predict(&config.embedding)?;

        // Compute acquisition function value
        let acquisition_value = match &self.bayesian_optimizer.acquisition_function {
            AcquisitionFunction::ExpectedImprovement { xi } => {
                let best_value = self.get_best_performance();
                let improvement = best_value - mean;
                let std_dev = variance.sqrt();

                if std_dev > 1e-8 {
                    let z = (improvement + xi) / std_dev;
                    improvement * self.normal_cdf(z) + std_dev * self.normal_pdf(z)
                } else {
                    0.0
                }
            }
            AcquisitionFunction::UpperConfidenceBound { beta } => mean + beta * variance.sqrt(),
            _ => mean + variance.sqrt(), // Default UCB
        };

        Ok(acquisition_value)
    }

    /// Normal CDF approximation
    fn normal_cdf(&self, x: f64) -> f64 {
        // Approximation of error function for Gaussian CDF
        // Using tanh approximation: erf(x) ≈ tanh(√(π/2) * x)
        let sqrt_pi_over_2 = (std::f64::consts::PI / 2.0).sqrt();
        0.5 * (1.0 + (sqrt_pi_over_2 * x / 2.0_f64.sqrt()).tanh())
    }

    /// Normal PDF
    fn normal_pdf(&self, x: f64) -> f64 {
        (1.0 / (2.0 * std::f64::consts::PI).sqrt()) * (-0.5 * x * x).exp()
    }

    /// Get best performance from database
    fn get_best_performance(&self) -> f64 {
        self.performance_database
            .records
            .iter()
            .map(|r| r.performance)
            .fold(f64::INFINITY, |a, b| a.min(b))
    }

    /// Select fidelity level for evaluation
    fn select_fidelity_level(
        &self,
        _config: &HyperparameterConfig,
        remaining_budget: f64,
    ) -> OptimizeResult<f64> {
        match &self.multi_fidelity_evaluator.selection_strategy {
            FidelitySelectionStrategy::Static(fidelity) => Ok(*fidelity),
            FidelitySelectionStrategy::Adaptive {
                initial_fidelity,
                adaptation_rate: _,
            } => {
                // Simple adaptive strategy based on remaining _budget
                let budget_ratio = remaining_budget / self.tuning_stats.total_cost.max(1.0);
                Ok(initial_fidelity * budget_ratio.max(0.1).min(1.0))
            }
            _ => Ok(0.5), // Default medium fidelity
        }
    }

    /// Update tuning statistics
    fn update_tuning_stats(&mut self, performance: f64, cost: f64) -> OptimizeResult<()> {
        self.tuning_stats.total_evaluations += 1;
        self.tuning_stats.total_cost += cost;

        if performance < self.tuning_stats.best_performance {
            self.tuning_stats.best_performance = performance;
        }

        // Update convergence rate (simplified)
        if self.tuning_stats.total_evaluations > 1 {
            let improvement_rate = (self.tuning_stats.best_performance - performance)
                / self.tuning_stats.total_evaluations as f64;
            self.tuning_stats.convergence_rate = improvement_rate.max(0.0);
        }

        Ok(())
    }

    /// Check convergence criteria
    fn check_convergence(&self) -> bool {
        // Simple convergence check
        self.tuning_stats.total_evaluations > 50 && self.tuning_stats.convergence_rate < 1e-6
    }

    /// Get tuning statistics
    pub fn get_tuning_stats(&self) -> &HyperparameterTuningStats {
        &self.tuning_stats
    }
}

impl HyperparameterSpace {
    /// Create default hyperparameter space for optimization
    pub fn create_default_space() -> Self {
        let continuous_params = vec![
            ContinuousHyperparameter {
                name: "learning_rate".to_string(),
                lower_bound: 1e-5,
                upper_bound: 1.0,
                scale: ParameterScale::Logarithmic,
                default_value: 0.01,
                importance_score: 1.0,
            },
            ContinuousHyperparameter {
                name: "momentum".to_string(),
                lower_bound: 0.0,
                upper_bound: 0.99,
                scale: ParameterScale::Linear,
                default_value: 0.9,
                importance_score: 0.8,
            },
            ContinuousHyperparameter {
                name: "weight_decay".to_string(),
                lower_bound: 1e-8,
                upper_bound: 1e-2,
                scale: ParameterScale::Logarithmic,
                default_value: 1e-4,
                importance_score: 0.6,
            },
        ];

        let discrete_params = vec![
            DiscreteHyperparameter {
                name: "max_nit".to_string(),
                values: vec![10, 50, 100, 500, 1000],
                default_value: 100,
                importance_score: 0.9,
            },
            DiscreteHyperparameter {
                name: "batch_size".to_string(),
                values: vec![1, 8, 16, 32, 64, 128],
                default_value: 32,
                importance_score: 0.7,
            },
        ];

        let categorical_params = vec![CategoricalHyperparameter {
            name: "optimizer_type".to_string(),
            categories: vec!["sgd".to_string(), "adam".to_string(), "lbfgs".to_string()],
            default_category: "adam".to_string(),
            category_embeddings: HashMap::new(),
            importance_score: 1.0,
        }];

        Self {
            continuous_params,
            discrete_params,
            categorical_params,
            conditional_dependencies: Vec::new(),
            parameter_bounds: HashMap::new(),
        }
    }
}

impl HyperparameterConfig {
    /// Create new hyperparameter configuration
    pub fn new(parameters: HashMap<String, ParameterValue>) -> Self {
        let config_hash = Self::compute_hash(&parameters);
        let embedding = Self::compute_embedding(&parameters);

        Self {
            parameters,
            config_hash,
            embedding,
        }
    }

    /// Compute hash for configuration
    fn compute_hash(parameters: &HashMap<String, ParameterValue>) -> u64 {
        // Simplified hash computation
        let mut hash = 0u64;
        for (key, value) in parameters {
            hash ^= Self::hash_string(key);
            hash ^= Self::hash_parameter_value(value);
        }
        hash
    }

    /// Hash string
    fn hash_string(s: &str) -> u64 {
        // Simple string hash
        s.bytes().fold(0u64, |hash, byte| {
            hash.wrapping_mul(31).wrapping_add(byte as u64)
        })
    }

    /// Hash parameter value
    fn hash_parameter_value(value: &ParameterValue) -> u64 {
        match value {
            ParameterValue::Continuous(v) => v.to_bits(),
            ParameterValue::Discrete(v) => *v as u64,
            ParameterValue::Categorical(s) => Self::hash_string(s),
        }
    }

    /// Compute embedding for configuration
    fn compute_embedding(parameters: &HashMap<String, ParameterValue>) -> Array1<f64> {
        let mut embedding = Array1::zeros(32); // Fixed embedding size

        let mut idx = 0;
        for value in parameters.values() {
            if idx >= embedding.len() {
                break;
            }

            match value {
                ParameterValue::Continuous(v) => {
                    embedding[idx] = v.tanh();
                    idx += 1;
                }
                ParameterValue::Discrete(v) => {
                    embedding[idx] = (*v as f64 / 100.0).tanh();
                    idx += 1;
                }
                ParameterValue::Categorical(s) => {
                    // Simple categorical encoding
                    let hash_val = Self::hash_string(s) as f64 / u64::MAX as f64;
                    embedding[idx] = (hash_val * 2.0 - 1.0).tanh();
                    idx += 1;
                }
            }
        }

        embedding
    }
}

impl Default for PerformanceDatabase {
    fn default() -> Self {
        Self::new()
    }
}

impl PerformanceDatabase {
    /// Create new performance database
    pub fn new() -> Self {
        Self {
            records: Vec::new(),
            index: HashMap::new(),
            performance_trends: HashMap::new(),
            correlation_matrix: Array2::zeros((0, 0)),
        }
    }

    /// Add evaluation record
    pub fn add_record(&mut self, record: EvaluationRecord) {
        self.records.push(record);

        // Update index (simplified)
        let record_idx = self.records.len() - 1;
        self.index
            .entry("all".to_string())
            .or_default()
            .push(record_idx);
    }
}

impl Default for BayesianOptimizer {
    fn default() -> Self {
        Self::new()
    }
}

impl BayesianOptimizer {
    /// Create new Bayesian optimizer
    pub fn new() -> Self {
        Self {
            gaussian_process: GaussianProcess::new(),
            acquisition_function: AcquisitionFunction::ExpectedImprovement { xi: 0.01 },
            optimization_strategy: OptimizationStrategy::RandomSearch {
                num_candidates: 100,
            },
            exploration_factor: 0.1,
        }
    }
}

impl Default for GaussianProcess {
    fn default() -> Self {
        Self::new()
    }
}

impl GaussianProcess {
    /// Create new Gaussian process
    pub fn new() -> Self {
        Self {
            training_inputs: Array2::zeros((0, 0)),
            training_outputs: Array1::zeros(0),
            kernel: KernelFunction::RBF {
                length_scale: 1.0,
                variance: 1.0,
            },
            kernel_params: Array1::from(vec![1.0, 1.0]),
            noise_variance: 0.1,
            mean_function: MeanFunction::Zero,
        }
    }

    /// Update training data
    pub fn update_training_data(
        &mut self,
        inputs: Array2<f64>,
        outputs: Array1<f64>,
    ) -> OptimizeResult<()> {
        self.training_inputs = inputs;
        self.training_outputs = outputs;
        Ok(())
    }

    /// Optimize hyperparameters
    pub fn optimize_hyperparameters(&mut self) -> OptimizeResult<()> {
        // Simplified hyperparameter optimization
        // In practice, would use marginal likelihood optimization
        Ok(())
    }

    /// Predict mean and variance
    pub fn predict(&self, input: &Array1<f64>) -> OptimizeResult<(f64, f64)> {
        if self.training_inputs.is_empty() {
            return Ok((0.0, 1.0));
        }

        // Simplified GP prediction
        let mean = 0.0; // Would compute proper posterior mean
        let variance = 1.0; // Would compute proper posterior variance

        Ok((mean, variance))
    }
}

impl Default for MultiFidelityEvaluator {
    fn default() -> Self {
        Self::new()
    }
}

impl MultiFidelityEvaluator {
    /// Create new multi-fidelity evaluator
    pub fn new() -> Self {
        let fidelity_levels = vec![
            FidelityLevel {
                fidelity: 0.1,
                cost_multiplier: 0.1,
                accuracy: 0.7,
                resource_requirements: ResourceRequirements {
                    computation_time: 1.0,
                    memory_usage: 0.5,
                    cpu_cores: 1,
                    gpu_required: false,
                },
            },
            FidelityLevel {
                fidelity: 0.5,
                cost_multiplier: 0.5,
                accuracy: 0.9,
                resource_requirements: ResourceRequirements {
                    computation_time: 5.0,
                    memory_usage: 1.0,
                    cpu_cores: 2,
                    gpu_required: false,
                },
            },
            FidelityLevel {
                fidelity: 1.0,
                cost_multiplier: 1.0,
                accuracy: 1.0,
                resource_requirements: ResourceRequirements {
                    computation_time: 10.0,
                    memory_usage: 2.0,
                    cpu_cores: 4,
                    gpu_required: true,
                },
            },
        ];

        Self {
            fidelity_levels,
            cost_model: CostModel::new(),
            selection_strategy: FidelitySelectionStrategy::Adaptive {
                initial_fidelity: 0.5,
                adaptation_rate: 0.1,
            },
            correlation_estimator: FidelityCorrelationEstimator::new(),
        }
    }
}

impl Default for CostModel {
    fn default() -> Self {
        Self::new()
    }
}

impl CostModel {
    /// Create new cost model
    pub fn new() -> Self {
        Self {
            cost_network: Array2::from_shape_fn((1, 10), |_| {
                (scirs2_core::random::rng().random::<f64>() - 0.5) * 0.1
            }),
            base_cost: 1.0,
            scaling_factors: Array1::ones(5),
            cost_history: VecDeque::with_capacity(1000),
        }
    }
}

impl Default for FidelityCorrelationEstimator {
    fn default() -> Self {
        Self::new()
    }
}

impl FidelityCorrelationEstimator {
    /// Create new correlation estimator
    pub fn new() -> Self {
        Self {
            correlation_matrix: Array2::eye(3),
            estimation_method: CorrelationMethod::Pearson,
            confidence_intervals: Array2::zeros((3, 2)),
        }
    }
}

impl Default for HyperparameterTuningStats {
    fn default() -> Self {
        Self {
            total_evaluations: 0,
            best_performance: f64::INFINITY,
            total_cost: 0.0,
            convergence_rate: 0.0,
            exploration_efficiency: 0.0,
            multi_fidelity_savings: 0.0,
        }
    }
}

impl LearnedOptimizer for LearnedHyperparameterTuner {
    fn meta_train(&mut self, training_tasks: &[TrainingTask]) -> OptimizeResult<()> {
        for task in training_tasks {
            // Create simple objective for training
            let training_objective = |x: &ArrayView1<f64>| x.iter().map(|&xi| xi * xi).sum::<f64>();

            let initial_params = Array1::zeros(task.problem.dimension);

            // Tune hyperparameters for this task
            let _best_config = self.tune_hyperparameters(
                training_objective,
                &initial_params.view(),
                &task.problem,
                10.0,
            )?;
        }

        Ok(())
    }

    fn adapt_to_problem(
        &mut self,
        problem: &OptimizationProblem,
        initial_params: &ArrayView1<f64>,
    ) -> OptimizeResult<()> {
        // Extract problem features for future configuration selection
        let simple_objective = |_x: &ArrayView1<f64>| 0.0;
        let _problem_features =
            self.extract_problem_features(&simple_objective, initial_params, problem)?;

        Ok(())
    }

    fn optimize<F>(
        &mut self,
        objective: F,
        initial_params: &ArrayView1<f64>,
    ) -> OptimizeResult<OptimizeResults<f64>>
    where
        F: Fn(&ArrayView1<f64>) -> f64,
    {
        // Create default problem for hyperparameter tuning
        let default_problem = OptimizationProblem {
            name: "hyperparameter_tuning".to_string(),
            dimension: initial_params.len(),
            problem_class: "general".to_string(),
            metadata: HashMap::new(),
            max_evaluations: 1000,
            target_accuracy: 1e-6,
        };

        // Tune hyperparameters
        let best_config =
            self.tune_hyperparameters(&objective, initial_params, &default_problem, 20.0)?;

        // Use best configuration for final optimization
        self.create_optimizer_from_config(&best_config, &objective, initial_params, 1.0)
    }

    fn get_state(&self) -> &MetaOptimizerState {
        &self.meta_state
    }

    fn reset(&mut self) {
        self.performance_database = PerformanceDatabase::new();
        self.tuning_stats = HyperparameterTuningStats::default();
    }
}

/// Convenience function for learned hyperparameter tuning
#[allow(dead_code)]
pub fn hyperparameter_tuning_optimize<F>(
    objective: F,
    initial_params: &ArrayView1<f64>,
    config: Option<LearnedOptimizationConfig>,
) -> super::OptimizeResult<OptimizeResults<f64>>
where
    F: Fn(&ArrayView1<f64>) -> f64,
{
    let config = config.unwrap_or_default();
    let mut tuner = LearnedHyperparameterTuner::new(config);
    tuner.optimize(objective, initial_params)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_hyperparameter_tuner_creation() {
        let config = LearnedOptimizationConfig::default();
        let tuner = LearnedHyperparameterTuner::new(config);

        assert_eq!(tuner.tuning_stats.total_evaluations, 0);
        assert!(!tuner.hyperparameter_space.continuous_params.is_empty());
    }

    #[test]
    fn test_hyperparameter_space() {
        let space = HyperparameterSpace::create_default_space();

        assert!(!space.continuous_params.is_empty());
        assert!(!space.discrete_params.is_empty());
        assert!(!space.categorical_params.is_empty());
    }

    #[test]
    fn test_hyperparameter_config() {
        let mut parameters = HashMap::new();
        parameters.insert(
            "learning_rate".to_string(),
            ParameterValue::Continuous(0.01),
        );
        parameters.insert("max_nit".to_string(), ParameterValue::Discrete(100));
        parameters.insert(
            "optimizer_type".to_string(),
            ParameterValue::Categorical("adam".to_string()),
        );

        let config = HyperparameterConfig::new(parameters);

        assert!(config.config_hash != 0);
        assert_eq!(config.embedding.len(), 32);
        assert!(config.embedding.iter().all(|&x| x.is_finite()));
    }

    #[test]
    fn test_problem_similarity() {
        let config = LearnedOptimizationConfig::default();
        let tuner = LearnedHyperparameterTuner::new(config);

        let features1 = Array1::from(vec![1.0, 0.0, 0.0]);
        let features2 = Array1::from(vec![0.0, 1.0, 0.0]);
        let features3 = Array1::from(vec![1.0, 0.1, 0.1]);

        let sim1 = tuner
            .compute_problem_similarity(&features1, &features2)
            .expect("Operation failed");
        let sim2 = tuner
            .compute_problem_similarity(&features1, &features3)
            .expect("Operation failed");

        assert!(sim2 > sim1); // features3 should be more similar to features1
    }

    #[test]
    fn test_gaussian_process() {
        let mut gp = GaussianProcess::new();

        let inputs = Array2::from_shape_fn((3, 2), |_| scirs2_core::random::rng().random::<f64>());
        let outputs = Array1::from(vec![1.0, 2.0, 3.0]);

        gp.update_training_data(inputs, outputs)
            .expect("Operation failed");

        let test_input = Array1::from(vec![0.5, 0.5]);
        let (mean, variance) = gp.predict(&test_input).expect("Operation failed");

        assert!(mean.is_finite());
        assert!(variance >= 0.0);
    }

    #[test]
    fn test_hyperparameter_tuning_optimization() {
        let objective = |x: &ArrayView1<f64>| x[0].powi(2) + x[1].powi(2);
        let initial = Array1::from(vec![2.0, 2.0]);

        let config = LearnedOptimizationConfig {
            hidden_size: 32,
            ..Default::default()
        };

        let result = hyperparameter_tuning_optimize(objective, &initial.view(), Some(config))
            .expect("Operation failed");

        assert!(result.fun >= 0.0);
        assert_eq!(result.x.len(), 2);
        assert!(result.success);
    }
}