mod compute;
#[cfg(test)]
mod tests;
const SHADER_F64: &str = include_str!("../../shaders/optimizer/sgd_f64.wgsl");
static SHADER_F32: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| SHADER_F64.to_string());
use crate::error::{BarracudaError, Result};
use crate::tensor::Tensor;
#[repr(C)]
#[derive(Copy, Clone, Debug, bytemuck::Pod, bytemuck::Zeroable)]
pub(crate) struct SGDParams {
pub learning_rate: f32,
pub momentum: f32,
pub weight_decay: f32,
pub dampening: f32,
}
pub struct SGD {
weights: Tensor,
gradients: Tensor,
velocity: Option<Tensor>,
learning_rate: f32,
momentum: f32,
weight_decay: f32,
}
impl SGD {
pub fn new(
weights: Tensor,
gradients: Tensor,
learning_rate: f32,
momentum: f32,
weight_decay: f32,
velocity: Option<Tensor>,
) -> Result<Self> {
if weights.shape() != gradients.shape() {
return Err(BarracudaError::shape_mismatch(
weights.shape().to_vec(),
gradients.shape().to_vec(),
));
}
if learning_rate <= 0.0 {
return Err(BarracudaError::invalid_op(
"sgd",
"learning_rate must be positive",
));
}
if !(0.0..=1.0).contains(&momentum) {
return Err(BarracudaError::invalid_op(
"sgd",
"momentum must be in range [0.0, 1.0]",
));
}
if weight_decay < 0.0 {
return Err(BarracudaError::invalid_op(
"sgd",
"weight_decay must be non-negative",
));
}
if let Some(ref v) = velocity
&& v.shape() != weights.shape()
{
return Err(BarracudaError::shape_mismatch(
v.shape().to_vec(),
weights.shape().to_vec(),
));
}
Ok(Self {
weights,
gradients,
velocity,
learning_rate,
momentum,
weight_decay,
})
}
pub(super) fn shader() -> &'static str {
&SHADER_F32
}
pub(super) fn weights(&self) -> &Tensor {
&self.weights
}
pub(super) fn gradients(&self) -> &Tensor {
&self.gradients
}
pub(super) fn velocity(&self) -> Option<&Tensor> {
self.velocity.as_ref()
}
pub(super) fn learning_rate(&self) -> f32 {
self.learning_rate
}
pub(super) fn momentum(&self) -> f32 {
self.momentum
}
pub(super) fn weight_decay(&self) -> f32 {
self.weight_decay
}
}
impl Tensor {
pub fn sgd_step(
self,
gradients: &Self,
learning_rate: f32,
momentum: f32,
weight_decay: f32,
velocity: Option<&Self>,
) -> Result<(Self, Option<Self>)> {
SGD::new(
self,
gradients.clone(),
learning_rate,
momentum,
weight_decay,
velocity.cloned(),
)?
.execute()
}
}