rust_ml/optim/sgd/
optimizer.rs

1/// Gradient Descent optimizer implementation for machine learning models.
2///
3/// This module provides an implementation of the gradient descent optim algorithm
4/// for training machine learning models. Gradient descent works by iteratively adjusting
5/// model parameters in the direction that minimizes the cost function.
6use crate::core::error::ModelError;
7use crate::core::types::{Matrix, Vector};
8use crate::model::core::base::OptimizableModel;
9use crate::optim::core::optimizer::Optimizer;
10use crate::optim::core::state::OptimizerState;
11use crate::optim::sgd::state::GradientDescentState;
12
13/// A standard gradient descent optimizer.
14///
15/// Gradient descent is an optim algorithm that iteratively adjusts parameters
16/// to minimize a cost function by moving in the direction of the steepest decrease
17/// in the cost function.
18///
19/// # Fields
20/// * `learning_rate` - The step size for each iteration of gradient descent
21/// * `epochs` - The number of complete passes through the training dataset
22/// * `cost_history` - Records the cost value after each parameter update
23pub struct GradientDescent<Input, Output, M: OptimizableModel<Input, Output>> {
24    epochs: usize,
25    pub cost_history: Vec<f64>,
26    state: GradientDescentState<Input, Output, M>,
27}
28
29impl<Input, Output, M: OptimizableModel<Input, Output>> GradientDescent<Input, Output, M> {
30    /// Creates a new GradientDescent optimizer.
31    ///
32    /// # Arguments
33    /// * `learning_rate` - The step size for each parameter update
34    /// * `epochs` - The number of complete passes through the training dataset
35    ///
36    /// # Returns
37    /// A new GradientDescent instance
38    pub fn new(learning_rate: f64, epochs: usize) -> Self {
39        Self {
40            epochs,
41            cost_history: Vec::new(),
42            state: GradientDescentState::new(learning_rate),
43        }
44    }
45}
46
47impl<M: OptimizableModel<Matrix, Vector>> Optimizer<Matrix, Vector, M>
48    for GradientDescent<Matrix, Vector, M>
49{
50    /// Fits the model to the training data using gradient descent algorithm.
51    ///
52    /// This method updates the model parameters by computing gradients
53    /// and adjusting the parameters in the direction that minimizes the cost function.
54    ///
55    /// # Arguments
56    /// * `model` - The machine learning model to optimize
57    /// * `x` - The input training data
58    /// * `y` - The expected output values
59    ///
60    /// # Returns
61    /// * `Ok(())` if optim completes successfully
62    /// * `Err(ModelError)` if an error occurs during optim
63    fn fit(&mut self, model: &mut M, x: &Matrix, y: &Vector) -> Result<(), ModelError> {
64        for _ in 0..self.epochs {
65            // Compute cost
66            let cost = model.compute_cost(x, y)?;
67
68            self.cost_history.push(cost);
69
70            // Compute output gradient (includes forward prop)
71            let output_gradient = model.compute_output_gradient(x, y)?;
72
73            // Compute gradients using backward propagation
74            model.backward(x, &output_gradient)?;
75
76            // Update model parameters using optimizer state
77            self.state.update_weights(model)?;
78        }
79        Ok(())
80    }
81}
rust_ml/optim/sgd/optimizer.rs

rust_ml/optim/sgd/
optimizer.rs
