Crate scirs2_optim

Machine Learning optimization module for SciRS2

This module provides optimization algorithms specifically designed for machine learning, including stochastic gradient descent variants, learning rate schedulers, and regularization techniques.

Features

• Optimization algorithms: SGD, Adam, RMSprop, etc.
• Learning rate schedulers: ExponentialDecay, CosineAnnealing, etc.
• Regularization techniques: L1, L2, Dropout, etc.

Examples

Traditional API

use ndarray::{Array1, Array2};
use scirs2_optim::optimizers::{SGD, Optimizer};

// Create a simple optimization problem
let params = Array1::zeros(5);
let gradients = Array1::from_vec(vec![0.1, 0.2, -0.3, 0.0, 0.5]);

// Create an optimizer with learning rate 0.01
let mut optimizer = SGD::new(0.01);

// Update parameters using the optimizer
let updated_params = optimizer.step(&params, &gradients);
// Parameters should be updated in the negative gradient direction

Unified API (PyTorch-style)

use ndarray::Array1;
use scirs2_optim::{OptimizerConfig, OptimizerFactory, Parameter, UnifiedOptimizer};

// Create parameters (similar to PyTorch)
let mut param1 = Parameter::new(Array1::from_vec(vec![1.0, 2.0, 3.0]), "layer1.weight");
let mut param2 = Parameter::new(Array1::from_vec(vec![0.1, 0.2]), "layer1.bias");

// Set gradients
param1.set_grad(Array1::from_vec(vec![0.1, 0.2, 0.3]));
param2.set_grad(Array1::from_vec(vec![0.05, 0.1]));

// Create optimizer with configuration
let config = OptimizerConfig::new(0.001f64)
    .weight_decay(0.0001)
    .grad_clip(1.0);
let mut optimizer = OptimizerFactory::adam(config);

// Update parameters
optimizer.step_param(&mut param1).unwrap();
optimizer.step_param(&mut param2).unwrap();

Re-exports

pub use adaptive_selection::AdaptiveOptimizerSelector;

pub use adaptive_selection::OptimizerStatistics;

pub use adaptive_selection::OptimizerType;

pub use adaptive_selection::PerformanceMetrics;

pub use adaptive_selection::ProblemCharacteristics;

pub use adaptive_selection::ProblemType;

pub use adaptive_selection::SelectionNetwork;

pub use adaptive_selection::SelectionStrategy;

pub use benchmarking::visualization::OptimizerDashboard;

pub use benchmarking::visualization::OptimizerStateSnapshot;

pub use benchmarking::visualization::OptimizerStateVisualizer;

pub use benchmarking::visualization::VisualizationExport;

pub use benchmarking::BenchmarkResult;

pub use benchmarking::GradientFlowAnalyzer;

pub use benchmarking::GradientFlowStats;

pub use benchmarking::OptimizerBenchmark;

pub use benchmarking::VisualizationData;

pub use curriculum_optimization::AdaptiveCurriculum;

pub use curriculum_optimization::AdversarialAttack;

pub use curriculum_optimization::AdversarialConfig;

pub use curriculum_optimization::CurriculumManager;

pub use curriculum_optimization::CurriculumState;

pub use curriculum_optimization::CurriculumStrategy;

pub use curriculum_optimization::ImportanceWeightingStrategy;

pub use distributed::AveragingStrategy;

pub use distributed::CommunicationResult;

pub use distributed::CompressedGradient;

pub use distributed::CompressionStrategy;

pub use distributed::DistributedCoordinator;

pub use distributed::GradientCompressor;

pub use distributed::ParameterAverager;

pub use distributed::ParameterServer;

pub use domain_specific::CrossDomainKnowledge;

pub use domain_specific::DomainOptimizationConfig;

pub use domain_specific::DomainPerformanceMetrics;

pub use domain_specific::DomainRecommendation;

pub use domain_specific::DomainSpecificSelector;

pub use domain_specific::DomainStrategy;

pub use domain_specific::LearningRateScheduleType;

pub use domain_specific::OptimizationContext;

pub use domain_specific::RecommendationType;

pub use domain_specific::RegularizationApproach;

pub use domain_specific::ResourceConstraints;

pub use domain_specific::TrainingConfiguration;

pub use gradient_accumulation::AccumulationMode;

pub use gradient_accumulation::GradientAccumulator as GradAccumulator;

pub use gradient_accumulation::MicroBatchTrainer;

pub use gradient_accumulation::VariableAccumulator;

pub use hardware_aware::AllReduceAlgorithm;

pub use hardware_aware::CommunicationStrategy;

pub use hardware_aware::GPUArchitecture;

pub use hardware_aware::HardwareAwareOptimizer;

pub use hardware_aware::HardwareOptimizationConfig;

pub use hardware_aware::HardwarePerformanceStats;

pub use hardware_aware::HardwarePlatform;

pub use hardware_aware::MemoryStrategy;

pub use hardware_aware::ParallelizationStrategy;

pub use hardware_aware::PartitionStrategy;

pub use hardware_aware::PerformanceProfiler;

pub use hardware_aware::PrecisionStrategy;

pub use hardware_aware::QuantizationSupport;

pub use hardware_aware::ResourceMonitor;

pub use hardware_aware::SIMDSupport;

pub use hardware_aware::TPUVersion;

pub use hardware_aware::TuningStrategy;

pub use meta_learning::AcquisitionFunction;

pub use meta_learning::HyperparameterOptimizer;

pub use meta_learning::HyperparameterPredictor;

pub use meta_learning::HyperparameterStrategy;

pub use meta_learning::MetaOptimizer;

pub use meta_learning::MetaOptimizerTrait;

pub use meta_learning::NeuralOptimizer;

pub use meta_learning::OptimizationTrajectory;

pub use meta_learning::SGDMetaOptimizer;

pub use meta_learning::UpdateNetwork;

pub use neural_integration::architecture_aware::ArchitectureAwareOptimizer;

pub use neural_integration::architecture_aware::ArchitectureStrategy;

pub use neural_integration::forward_backward::BackwardHook;

pub use neural_integration::forward_backward::ForwardHook;

pub use neural_integration::forward_backward::NeuralIntegration;

pub use neural_integration::LayerArchitecture;

pub use neural_integration::LayerId;

pub use neural_integration::OptimizationConfig;

pub use neural_integration::ParamId;

pub use neural_integration::ParameterManager;

pub use neural_integration::ParameterMetadata;

pub use neural_integration::ParameterOptimizer;

pub use neural_integration::ParameterType;

pub use online_learning::ColumnGrowthStrategy;

pub use online_learning::LearningRateAdaptation;

pub use online_learning::LifelongOptimizer;

pub use online_learning::LifelongStats;

pub use online_learning::LifelongStrategy;

pub use online_learning::MemoryExample;

pub use online_learning::MemoryUpdateStrategy;

pub use online_learning::MirrorFunction;

pub use online_learning::OnlineLearningStrategy;

pub use online_learning::OnlineOptimizer;

pub use online_learning::OnlinePerformanceMetrics;

pub use online_learning::SharedKnowledge;

pub use online_learning::TaskGraph;

pub use second_order::HessianInfo;

pub use second_order::Newton;

pub use second_order::SecondOrderOptimizer;

pub use second_order::LBFGS as SecondOrderLBFGS;

pub use training_stabilization::AveragingMethod;

pub use training_stabilization::ModelEnsemble;

pub use training_stabilization::PolyakAverager;

pub use training_stabilization::WeightAverager;

pub use unified_api::OptimizerConfig;

pub use unified_api::OptimizerFactory;

pub use unified_api::Parameter;

pub use unified_api::TrainingLoop;

pub use unified_api::UnifiedAdam;

pub use unified_api::UnifiedOptimizer;

pub use unified_api::UnifiedSGD;

pub use gradient_processing::*;

pub use optimizer_composition::*;

pub use optimizers::*;

pub use regularizers::*;

pub use schedulers::*;

Modules

adaptive_selection

Adaptive optimization algorithm selection

benchmarking

Benchmarking and evaluation tools for optimizers

curriculum_optimization

Curriculum optimization for adaptive training

distributed

Distributed optimization support

domain_specific

Domain-specific optimization strategies

error

Error types for the ML optimization module

gradient_accumulation

Gradient accumulation for large batch training

gradient_processing

Gradient processing utilities for machine learning optimization

hardware_aware

Hardware-aware optimization routines

memory_efficient

Memory-efficient optimizers and utilities

meta_learning

Meta-learning optimization support

metrics

Metrics integration for optimization

neural_integration

Neural network integration for optimizers

online_learning

Online learning and lifelong optimization

optimizer_composition

Optimizer composition framework

optimizers

Optimization algorithms for machine learning

parameter_groups

Parameter groups for different learning rates and configurations

regularizers

Regularization techniques for machine learning

schedulers

Learning rate schedulers for optimizers

second_order

Second-order optimization methods

training_stabilization

Training stabilization techniques

unified_api

Unified API consistent with popular deep learning frameworks

utils

Utility functions for machine learning optimization