mod compute;
#[cfg(test)]
mod tests;
use crate::error::{BarracudaError, Result};
use crate::tensor::Tensor;
#[derive(Debug, Clone, Copy)]
pub struct AdamConfig {
pub learning_rate: f32,
pub beta1: f32,
pub beta2: f32,
pub epsilon: f32,
pub weight_decay: f32,
}
impl AdamConfig {
#[must_use]
pub fn new(learning_rate: f32) -> Self {
Self {
learning_rate,
beta1: 0.9,
beta2: 0.999,
epsilon: 1e-8,
weight_decay: 0.0,
}
}
#[must_use]
pub fn beta(mut self, beta1: f32, beta2: f32) -> Self {
self.beta1 = beta1;
self.beta2 = beta2;
self
}
#[must_use]
pub fn epsilon(mut self, epsilon: f32) -> Self {
self.epsilon = epsilon;
self
}
#[must_use]
pub fn weight_decay(mut self, weight_decay: f32) -> Self {
self.weight_decay = weight_decay;
self
}
pub(crate) fn validate(&self, op_name: &str) -> Result<()> {
if !(0.0..1.0).contains(&self.beta1) {
return Err(BarracudaError::invalid_op(
op_name,
format!("beta1 must be in [0, 1), got {}", self.beta1),
));
}
if !(0.0..1.0).contains(&self.beta2) {
return Err(BarracudaError::invalid_op(
op_name,
format!("beta2 must be in [0, 1), got {}", self.beta2),
));
}
if self.epsilon <= 0.0 {
return Err(BarracudaError::invalid_op(
op_name,
format!("epsilon must be positive, got {}", self.epsilon),
));
}
Ok(())
}
}
#[repr(C)]
#[derive(Copy, Clone, Debug, bytemuck::Pod, bytemuck::Zeroable)]
pub(crate) struct AdamWParams {
pub num_params: u32,
pub learning_rate: f32,
pub beta1: f32,
pub beta2: f32,
pub epsilon: f32,
pub weight_decay: f32,
pub step: u32,
}
pub struct AdamW {
params: Tensor,
gradients: Tensor,
m: Tensor,
v: Tensor,
config: AdamConfig,
step: u32,
}
impl AdamW {
pub fn new(
params: Tensor,
gradients: Tensor,
m: Tensor,
v: Tensor,
config: &AdamConfig,
step: u32,
) -> Result<Self> {
if params.shape() != gradients.shape() {
return Err(BarracudaError::shape_mismatch(
params.shape().to_vec(),
gradients.shape().to_vec(),
));
}
if params.shape() != m.shape() {
return Err(BarracudaError::shape_mismatch(
params.shape().to_vec(),
m.shape().to_vec(),
));
}
if params.shape() != v.shape() {
return Err(BarracudaError::shape_mismatch(
params.shape().to_vec(),
v.shape().to_vec(),
));
}
config.validate("AdamW")?;
if step == 0 {
return Err(BarracudaError::invalid_op(
"AdamW",
"step must be >= 1 for bias correction",
));
}
Ok(Self {
params,
gradients,
m,
v,
config: *config,
step,
})
}
pub(super) fn shader() -> &'static str {
{
static SHADER: std::sync::LazyLock<String> = std::sync::LazyLock::new(|| {
include_str!("../../shaders/optimizer/adamw_f64.wgsl").to_string()
});
std::sync::LazyLock::force(&SHADER).as_str()
}
}
pub(super) fn params(&self) -> &Tensor {
&self.params
}
pub(super) fn gradients(&self) -> &Tensor {
&self.gradients
}
pub(super) fn m(&self) -> &Tensor {
&self.m
}
pub(super) fn v(&self) -> &Tensor {
&self.v
}
pub(super) fn learning_rate(&self) -> f32 {
self.config.learning_rate
}
pub(super) fn beta1(&self) -> f32 {
self.config.beta1
}
pub(super) fn beta2(&self) -> f32 {
self.config.beta2
}
pub(super) fn epsilon(&self) -> f32 {
self.config.epsilon
}
pub(super) fn weight_decay(&self) -> f32 {
self.config.weight_decay
}
pub(super) fn step(&self) -> u32 {
self.step
}
}
impl Tensor {
pub fn adamw(
self,
gradients: &Self,
m: &Self,
v: &Self,
config: &AdamConfig,
step: u32,
) -> Result<(Self, Self, Self)> {
AdamW::new(self, gradients.clone(), m.clone(), v.clone(), config, step)?.execute()
}
}