use coeus_core::{ComputeBackend, Float, Layout, Scalar};
#[derive(Clone, Copy, Debug)]
pub enum OptimizerStepRule<T> {
Sgd {
learning_rate: T,
momentum: T,
},
Adam {
learning_rate: T,
beta_one: T,
beta_two: T,
epsilon: T,
step: usize,
},
RmsProp {
learning_rate: T,
alpha: T,
epsilon: T,
},
AdamW {
learning_rate: T,
beta_one: T,
beta_two: T,
epsilon: T,
weight_decay: T,
step: usize,
},
AdaGrad {
learning_rate: T,
epsilon: T,
},
}
pub enum OptimizerStateRef<'a, T: Scalar, B: ComputeBackend> {
One(&'a B::DeviceBuffer<T>, &'a Layout),
Two(
&'a B::DeviceBuffer<T>,
&'a Layout,
&'a B::DeviceBuffer<T>,
&'a Layout,
),
}
pub struct OptimizerStepValidation<'a, T: Scalar, B: ComputeBackend> {
pub parameter: (&'a B::DeviceBuffer<T>, &'a Layout),
pub gradient: (&'a B::DeviceBuffer<T>, &'a Layout),
pub state: OptimizerStateRef<'a, T, B>,
pub rule: OptimizerStepRule<T>,
}
pub trait OptimizerOps<T: Scalar>: ComputeBackend {
fn validate_optimizer_step(
&self,
validation: OptimizerStepValidation<'_, T, Self>,
) -> Result<(), Self::Error>
where
Self: Sized,
T: Float;
fn sgd_step(
&self,
param: &mut Self::DeviceBuffer<T>,
param_layout: &Layout,
grad: &Self::DeviceBuffer<T>,
grad_layout: &Layout,
velocity: &mut Self::DeviceBuffer<T>,
velocity_layout: &Layout,
lr: T,
momentum: T,
) -> Result<(), Self::Error>
where
T: Float;
fn adam_step(
&self,
param: &mut Self::DeviceBuffer<T>,
param_layout: &Layout,
grad: &Self::DeviceBuffer<T>,
grad_layout: &Layout,
m: &mut Self::DeviceBuffer<T>,
m_layout: &Layout,
v: &mut Self::DeviceBuffer<T>,
v_layout: &Layout,
lr: T,
beta1: T,
beta2: T,
eps: T,
t: usize,
) -> Result<(), Self::Error>
where
T: Float;
fn rmsprop_step(
&self,
param: &mut Self::DeviceBuffer<T>,
param_layout: &Layout,
grad: &Self::DeviceBuffer<T>,
grad_layout: &Layout,
v: &mut Self::DeviceBuffer<T>,
v_layout: &Layout,
lr: T,
alpha: T,
eps: T,
) -> Result<(), Self::Error>
where
T: Float;
fn adamw_step(
&self,
param: &mut Self::DeviceBuffer<T>,
param_layout: &Layout,
grad: &Self::DeviceBuffer<T>,
grad_layout: &Layout,
m: &mut Self::DeviceBuffer<T>,
m_layout: &Layout,
v: &mut Self::DeviceBuffer<T>,
v_layout: &Layout,
lr: T,
beta1: T,
beta2: T,
eps: T,
weight_decay: T,
t: usize,
) -> Result<(), Self::Error>
where
T: Float;
fn adagrad_step(
&self,
param: &mut Self::DeviceBuffer<T>,
param_layout: &Layout,
grad: &Self::DeviceBuffer<T>,
grad_layout: &Layout,
history: &mut Self::DeviceBuffer<T>,
history_layout: &Layout,
lr: T,
eps: T,
) -> Result<(), Self::Error>
where
T: Float;
}