use eunomia::{FloatElement, NumericElement};
use leto::{LetoError, Result};
use crate::RealScalar;
fn invalid(reason: &'static str) -> LetoError {
LetoError::InvalidInput(reason.to_string())
}
fn positive<T: RealScalar>(value: T, reason: &'static str) -> Result<()> {
if value.is_finite() && value > <T as NumericElement>::ZERO {
Ok(())
} else {
Err(invalid(reason))
}
}
fn nonnegative<T: RealScalar>(value: T, reason: &'static str) -> Result<()> {
if value.is_finite() && value >= <T as NumericElement>::ZERO {
Ok(())
} else {
Err(invalid(reason))
}
}
fn unit_interval<T: RealScalar>(value: T, reason: &'static str) -> Result<()> {
if value.is_finite()
&& value >= <T as NumericElement>::ZERO
&& value < <T as NumericElement>::ONE
{
Ok(())
} else {
Err(invalid(reason))
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct SgdParameters<T> {
pub(super) learning_rate: T,
pub(super) momentum: T,
}
impl<T: RealScalar> SgdParameters<T> {
pub fn new(learning_rate: T, momentum: T) -> Result<Self> {
nonnegative(
learning_rate,
"SGD learning rate must be finite and non-negative",
)?;
unit_interval(momentum, "SGD momentum must be finite in [0, 1)")?;
Ok(Self {
learning_rate,
momentum,
})
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct AdamParameters<T> {
pub(super) learning_rate: T,
pub(super) beta_one: T,
pub(super) beta_two: T,
pub(super) epsilon: T,
pub(super) bias_correction_one: T,
pub(super) bias_correction_two: T,
}
impl<T: RealScalar> AdamParameters<T> {
pub fn new(learning_rate: T, beta_one: T, beta_two: T, epsilon: T, step: u64) -> Result<Self> {
nonnegative(
learning_rate,
"Adam learning rate must be finite and non-negative",
)?;
unit_interval(beta_one, "Adam beta one must be finite in [0, 1)")?;
unit_interval(beta_two, "Adam beta two must be finite in [0, 1)")?;
positive(epsilon, "Adam epsilon must be finite and positive")?;
if step == 0 {
return Err(invalid("Adam step must be positive"));
}
let one = <T as NumericElement>::ONE;
let bias_correction_one = one - pow_unsigned(beta_one, step);
let bias_correction_two = one - pow_unsigned(beta_two, step);
positive(
bias_correction_one,
"Adam first bias correction must be positive",
)?;
positive(
bias_correction_two,
"Adam second bias correction must be positive",
)?;
Ok(Self {
learning_rate,
beta_one,
beta_two,
epsilon,
bias_correction_one,
bias_correction_two,
})
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct AdamWParameters<T> {
pub(super) adam: AdamParameters<T>,
pub(super) weight_decay: T,
}
impl<T: RealScalar> AdamWParameters<T> {
pub fn new(
learning_rate: T,
beta_one: T,
beta_two: T,
epsilon: T,
weight_decay: T,
step: u64,
) -> Result<Self> {
if !weight_decay.is_finite() || weight_decay < <T as NumericElement>::ZERO {
return Err(invalid(
"AdamW weight decay must be finite and non-negative",
));
}
Ok(Self {
adam: AdamParameters::new(learning_rate, beta_one, beta_two, epsilon, step)?,
weight_decay,
})
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct RmsPropParameters<T> {
pub(super) learning_rate: T,
pub(super) alpha: T,
pub(super) epsilon: T,
}
impl<T: RealScalar> RmsPropParameters<T> {
pub fn new(learning_rate: T, alpha: T, epsilon: T) -> Result<Self> {
nonnegative(
learning_rate,
"RMSProp learning rate must be finite and non-negative",
)?;
unit_interval(alpha, "RMSProp alpha must be finite in [0, 1)")?;
positive(epsilon, "RMSProp epsilon must be finite and positive")?;
Ok(Self {
learning_rate,
alpha,
epsilon,
})
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct AdaGradParameters<T> {
pub(super) learning_rate: T,
pub(super) epsilon: T,
}
impl<T: RealScalar> AdaGradParameters<T> {
pub fn new(learning_rate: T, epsilon: T) -> Result<Self> {
nonnegative(
learning_rate,
"AdaGrad learning rate must be finite and non-negative",
)?;
positive(epsilon, "AdaGrad epsilon must be finite and positive")?;
Ok(Self {
learning_rate,
epsilon,
})
}
}
fn pow_unsigned<T: FloatElement>(mut base: T, mut exponent: u64) -> T {
let mut result = <T as NumericElement>::ONE;
while exponent != 0 {
if exponent & 1 == 1 {
result *= base;
}
exponent >>= 1;
if exponent != 0 {
base *= base;
}
}
result
}